錯誤sample code 如下
/* * NCHC Hbase with map reduce sample code * DemoHBaseSlink.java */ package tw.org.nchc.demo; import java.io.IOException; import java.util.Iterator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapred.TableReduce; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; /** * This sample code will put the indicate data to Hbase. * 1. put test.txt in t1 directory which content is --------------- name locate years waue taiwan 1981 shellon taiwan 1981 --------------- * 2. hadoop_root/$ bin/hadoop dfs -put t1 t1 * 3. hbase_root/$ bin/hbase shell * 4. hql > create table t1_table("person"); * 5. run this code, and we will let database as that t1_table -> person ---------------- | name | locate | years | ---------------- | waue | taiwan | 1981 | ---------------- | shellon | taiwan | 1981 | **/ public class DemoHBaseSink { private static class ReduceClass extends TableReduce<LongWritable, Text> { // this is the column we're going to be writing private static final Text col1 = new Text("person:name"); private static final Text col2 = new Text("person:locate"); private static final Text col3 = new Text("person:years"); // this map holds the columns per row private MapWritable map = new MapWritable(); public void reduce(LongWritable key, Iterator<Text> values, OutputCollector<Text, MapWritable> output, Reporter reporter) throws IOException { // contents must be ImmutableBytesWritable String[] str = (values.next().getBytes().toString()).split(" "); byte bl[] = str[0].getBytes(); byte bn[] = str[1].getBytes(); byte by[] = str[2].getBytes(); ImmutableBytesWritable bytes1 = new ImmutableBytesWritable( bl); ImmutableBytesWritable bytes2 = new ImmutableBytesWritable( bn ); ImmutableBytesWritable bytes3 = new ImmutableBytesWritable( by ); // ImmutableBytesWritable bytes3 = new ImmutableBytesWritable(values.next().getBytes()); // populate the current row map.clear(); map.put(col1, bytes1); map.put(col2, bytes2); map.put(col3, bytes3); // add the row with the key as the row id output.collect(new Text(key.toString()), map); } } private DemoHBaseSink() { } /** * Runs the demo. */ public static void main(String[] args) throws IOException { // which path of input files in Hadoop file system String file_path = "/user/waue/t1"; int mapTasks = 1; int reduceTasks = 1; JobConf conf = new JobConf(DemoHBaseSink.class); conf.setJobName("DemoPersonBase"); // must initialize the TableReduce before running job TableReduce.initJob("t1_table", ReduceClass.class, conf); conf.setNumMapTasks(mapTasks); conf.setNumReduceTasks(reduceTasks); conf.setInputPath(new Path(file_path)); conf.setMapperClass(IdentityMapper.class); conf.setCombinerClass(IdentityReducer.class); conf.setReducerClass(ReduceClass.class); JobClient.runJob(conf); } }
錯誤訊息
08/06/04 18:03:21 INFO mapred.FileInputFormat: Total input paths to process : 1 08/06/04 18:03:22 INFO mapred.JobClient: Running job: job_200805291341_0019 08/06/04 18:03:23 INFO mapred.JobClient: map 0% reduce 0% 08/06/04 18:03:28 INFO mapred.JobClient: map 100% reduce 0% 08/06/04 18:03:36 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_0, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:40 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_1, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:45 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_2, Status : FAILED java.lang.ArrayIndexOutOfBoundsException: 1 at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63) at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333) at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084) 08/06/04 18:03:49 INFO mapred.JobClient: map 100% reduce 100% Exception in thread "main" java.io.IOException: Job failed! at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:892) at tw.org.nchc.demo.DemoHBaseSink.main(DemoHBaseSink.java:108) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) at java.lang.reflect.Method.invoke(Method.java:597) at org.apache.hadoop.util.RunJar.main(RunJar.java:155)
Last modified 17 years ago
Last modified on Jun 4, 2008, 6:11:56 PM