wiki:hadoop_hbase_error_sample2

Version 1 (modified by waue, 16 years ago) (diff)

--

錯誤sample code 如下

/*
 *  NCHC Hbase with map reduce sample code 
 *  DemoHBaseSlink.java
 */

package tw.org.nchc.demo;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapred.TableReduce;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;

/**
 * This sample code will put the indicate data to Hbase.
 * 1. put test.txt in t1 directory which content is 
---------------
name locate years 
waue taiwan 1981
shellon taiwan 1981
---------------
 * 2. hadoop_root/$ bin/hadoop dfs -put t1 t1
 * 3. hbase_root/$ bin/hbase shell
 * 4. hql > create table t1_table("person");
 * 5. run this code, and we will let database as that 
 t1_table -> person
  ----------------
  |  name | locate | years |
  ----------------
  | waue  | taiwan | 1981 |
  ----------------
  | shellon | taiwan | 1981 |
  
 **/
public class DemoHBaseSink {

	private static class ReduceClass extends TableReduce<LongWritable, Text> {

		// this is the column we're going to be writing
		private static final Text col1 = new Text("person:name");
		private static final Text col2 = new Text("person:locate");
		private static final Text col3 = new Text("person:years");
		// this map holds the columns per row
		private MapWritable map = new MapWritable();	

		public void reduce(LongWritable key, Iterator<Text> values,
				OutputCollector<Text, MapWritable> output, Reporter reporter)
				throws IOException {

			// contents must be ImmutableBytesWritable
			String[] str = (values.next().getBytes().toString()).split(" ");
			byte bl[] = str[0].getBytes();
			byte bn[] = str[1].getBytes();
			byte by[] = str[2].getBytes();
			ImmutableBytesWritable bytes1 = new ImmutableBytesWritable( bl);
			ImmutableBytesWritable bytes2 = new ImmutableBytesWritable( bn );
			ImmutableBytesWritable bytes3 = new ImmutableBytesWritable( by );
//			ImmutableBytesWritable bytes3 = new ImmutableBytesWritable(values.next().getBytes());
			// populate the current row
			map.clear();
			map.put(col1, bytes1);
			map.put(col2, bytes2);
			map.put(col3, bytes3);

			// add the row with the key as the row id
			output.collect(new Text(key.toString()), map);
		}
	}

	private DemoHBaseSink() {
	}

	/**
	 * Runs the demo.
	 */
	public static void main(String[] args) throws IOException {
		// which path of input files in Hadoop file system 
		String file_path = "/user/waue/t1";

		int mapTasks = 1;
		int reduceTasks = 1;

		JobConf conf = new JobConf(DemoHBaseSink.class);
		conf.setJobName("DemoPersonBase");

		// must initialize the TableReduce before running job
		TableReduce.initJob("t1_table", ReduceClass.class, conf);

		conf.setNumMapTasks(mapTasks);
		conf.setNumReduceTasks(reduceTasks);

		conf.setInputPath(new Path(file_path));

		conf.setMapperClass(IdentityMapper.class);
		conf.setCombinerClass(IdentityReducer.class);
		conf.setReducerClass(ReduceClass.class);

		JobClient.runJob(conf);
	}
}

錯誤訊息

08/06/04 18:03:21 INFO mapred.FileInputFormat: Total input paths to process : 1
08/06/04 18:03:22 INFO mapred.JobClient: Running job: job_200805291341_0019
08/06/04 18:03:23 INFO mapred.JobClient:  map 0% reduce 0%
08/06/04 18:03:28 INFO mapred.JobClient:  map 100% reduce 0%
08/06/04 18:03:36 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_0, Status : FAILED
java.lang.ArrayIndexOutOfBoundsException: 1
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
	at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
	at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)

08/06/04 18:03:40 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_1, Status : FAILED
java.lang.ArrayIndexOutOfBoundsException: 1
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
	at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
	at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)

08/06/04 18:03:45 INFO mapred.JobClient: Task Id : task_200805291341_0019_r_000000_2, Status : FAILED
java.lang.ArrayIndexOutOfBoundsException: 1
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:63)
	at tw.org.nchc.demo.DemoHBaseSink$ReduceClass.reduce(DemoHBaseSink.java:1)
	at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:333)
	at org.apache.hadoop.mapred.TaskTracker$Child.main(TaskTracker.java:2084)

08/06/04 18:03:49 INFO mapred.JobClient:  map 100% reduce 100%
Exception in thread "main" java.io.IOException: Job failed!
	at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:892)
	at tw.org.nchc.demo.DemoHBaseSink.main(DemoHBaseSink.java:108)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at org.apache.hadoop.util.RunJar.main(RunJar.java:155)