Changes between Initial Version and Version 1 of hadoop_hbase_error_sample2

Timestamp:

Jun 4, 2008, 6:11:56 PM (17 years ago)

Author:

waue

Comment:

--

hadoop_hbase_error_sample2

@@ +1 @@
+錯誤sample code 如下
+
+{{{
+/*
+ *  NCHC Hbase with map reduce sample code 
+ *  DemoHBaseSlink.java
+ */
+
+package tw.org.nchc.demo;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.mapred.TableReduce;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.MapWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.lib.IdentityMapper;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
+
+/**
+ * This sample code will put the indicate data to Hbase.
+ * 1. put test.txt in t1 directory which content is 
+---------------
+name locate years 
+waue taiwan 1981
+shellon taiwan 1981
+---------------
+ * 2. hadoop_root/$ bin/hadoop dfs -put t1 t1
+ * 3. hbase_root/$ bin/hbase shell
+ * 4. hql > create table t1_table("person");
+ * 5. run this code, and we will let database as that 
+ t1_table -> person
+  ----------------
+  |  name | locate | years |
+  ----------------
+  | waue  | taiwan | 1981 |
+  ----------------
+  | shellon | taiwan | 1981 |
+  
+ **/
+public class DemoHBaseSink {
+
+        private static class ReduceClass extends TableReduce<LongWritable, Text> {
+
+                // this is the column we're going to be writing
+                private static final Text col1 = new Text("person:name");
+                private static final Text col2 = new Text("person:locate");
+                private static final Text col3 = new Text("person:years");
+                // this map holds the columns per row
+                private MapWritable map = new MapWritable();    
+
+                public void reduce(LongWritable key, Iterator<Text> values,
+                                OutputCollector<Text, MapWritable> output, Reporter reporter)
+                                throws IOException {
+
+                        // contents must be ImmutableBytesWritable
+                        String[] str = (values.next().getBytes().toString()).split(" ");
+                        byte bl[] = str[0].getBytes();
+                        byte bn[] = str[1].getBytes();
+                        byte by[] = str[2].getBytes();
+                        ImmutableBytesWritable bytes1 = new ImmutableBytesWritable( bl);
+                        ImmutableBytesWritable bytes2 = new ImmutableBytesWritable( bn );
+                        ImmutableBytesWritable bytes3 = new ImmutableBytesWritable( by );
+//                      ImmutableBytesWritable bytes3 = new ImmutableBytesWritable(values.next().getBytes());
+                        // populate the current row
+                        map.clear();
+                        map.put(col1, bytes1);
+                        map.put(col2, bytes2);
+                        map.put(col3, bytes3);
+
+                        // add the row with the key as the row id
+                        output.collect(new Text(key.toString()), map);
+                }
+        }
+
+        private DemoHBaseSink() {
+        }
+
+        /**
+         * Runs the demo.
+         */
+        public static void main(String[] args) throws IOException {
+                // which path of input files in Hadoop file system 
+                String file_path = "/user/waue/t1";
+
+                int mapTasks = 1;
+                int reduceTasks = 1;
+
+                JobConf conf = new JobConf(DemoHBaseSink.class);
+                conf.setJobName("DemoPersonBase");
+
+                // must initialize the TableReduce before running job
+                TableReduce.initJob("t1_table", ReduceClass.class, conf);
+
+                conf.setNumMapTasks(mapTasks);
+                conf.setNumReduceTasks(reduceTasks);
+
+                conf.setInputPath(new Path(file_path));
+
+                conf.setMapperClass(IdentityMapper.class);
+                conf.setCombinerClass(IdentityReducer.class);
+                conf.setReducerClass(ReduceClass.class);
+
+                JobClient.runJob(conf);
+        }
+}
+}}}
+
+錯誤訊息
+{{{
+08/06/04 18:03:21 INFO mapred.FileInputFormat: Total input paths to process : 1
+08/06/04 18:03:22 INFO mapred.JobClient: Running job: job_200805291341_0019
+08/06/04 18:03:23 INFO mapred.JobClient:  map 0% reduce 0%
+08/06/04 18:03:28 INFO mapred.JobClient:  map 100% reduce 0%
+08/06/04 18:03:36 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_0, Status : FAILED
+java.lang.ArrayIndexOutOfBoundsException: 1
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
+        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
+        at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)
+
+08/06/04 18:03:40 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_1, Status : FAILED
+java.lang.ArrayIndexOutOfBoundsException: 1
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
+        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
+        at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)
+
+08/06/04 18:03:45 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_2, Status : FAILED
+java.lang.ArrayIndexOutOfBoundsException: 1
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
+        at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
+        at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
+        at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)
+
+08/06/04 18:03:49 INFO mapred.JobClient:  map 100% reduce 100%
+Exception in thread "main" java.io.IOException: Job failed!
+        at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:892)
+        at tw.org.nchc.demo.DemoHBaseSink.main(DemoHBaseSink.java:108)
+        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
+        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
+        at java.lang.reflect.Method.invoke(Method.java:597)
+        at org.apache.hadoop.util.RunJar.main(RunJar.java:155)
+
+}}}