Changeset 18


Ignore:
Timestamp:
Jul 2, 2008, 3:10:09 PM (16 years ago)
Author:
waue
Message:

upgrade 0.16 to 0.17

Location:
sample
Files:
6 edited

Legend:

Unmodified
Added
Removed

  • sample/HBaseRecord.java

    r9 r18  
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/01/2008
     5 * Last Update Date: 07/02/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    1314 *  Make sure Hadoop file system and Hbase are running correctly.
    1415 *  1. put test.txt in t1 directory which content is
    15   ---------------
    16   name:locate:years
    17   waue:taiwan:1981
    18   shellon:taiwan:1981
    19   ---------------
     16 ---------------
     17 name:locate:years
     18 waue:taiwan:1981
     19 shellon:taiwan:1981
     20 ---------------
    2021 *  2. hadoop_root/$ bin/hadoop dfs -put t1 t1
    2122 *  3. hbase_root/$ bin/hbase shell
    2223 *  4. hql > create table t1_table("person");
    2324 *  5. Come to Eclipse and run this code, and we will let database as that
    24   t1_table -> person
    25     ----------------
    26     |  name | locate | years |
    27     | waue  | taiwan | 1981 |
    28     | shellon | taiwan | 1981 |
    29     ----------------
     25 t1_table -> person
     26 ----------------
     27 |  name | locate | years |
     28 | waue  | taiwan | 1981 |
     29 | shellon | taiwan | 1981 |
     30 ----------------
    3031 * Check Result:
    3132 *  Go to hbase console, type :
    3233 *    hql > select * from t1_table;
    33 08/06/06 12:20:48 INFO hbase.HTable: Creating scanner over t1_table starting at key
    34 +-------------------------+-------------------------+-------------------------+
    35 | Row                     | Column                  | Cell                    |
    36 +-------------------------+-------------------------+-------------------------+
    37 | 0                       | person:locate           | locate                  |
    38 +-------------------------+-------------------------+-------------------------+
    39 | 0                       | person:name             | name                    |
    40 +-------------------------+-------------------------+-------------------------+
    41 | 0                       | person:years            | years                   |
    42 +-------------------------+-------------------------+-------------------------+
    43 | 19                      | person:locate           | taiwan                  |
    44 +-------------------------+-------------------------+-------------------------+
    45 | 19                      | person:name             | waue                    |
    46 +-------------------------+-------------------------+-------------------------+
    47 | 19                      | person:years            | 1981                    |
    48 +-------------------------+-------------------------+-------------------------+
    49 | 36                      | person:locate           | taiwan                  |
    50 +-------------------------+-------------------------+-------------------------+
    51 | 36                      | person:name             | shellon                 |
    52 +-------------------------+-------------------------+-------------------------+
    53 | 36                      | person:years            | 1981                    |
    54 +-------------------------+-------------------------+-------------------------+
    55 3 row(s) in set. (0.04 sec)
     34 08/06/06 12:20:48 INFO hbase.HTable: Creating scanner over t1_table starting at key
     35 +-------------------------+-------------------------+-------------------------+
     36 | Row                     | Column                  | Cell                    |
     37 +-------------------------+-------------------------+-------------------------+
     38 | 0                       | person:locate           | locate                  |
     39 +-------------------------+-------------------------+-------------------------+
     40 | 0                       | person:name             | name                    |
     41 +-------------------------+-------------------------+-------------------------+
     42 | 0                       | person:years            | years                   |
     43 +-------------------------+-------------------------+-------------------------+
     44 | 19                      | person:locate           | taiwan                  |
     45 +-------------------------+-------------------------+-------------------------+
     46 | 19                      | person:name             | waue                    |
     47 +-------------------------+-------------------------+-------------------------+
     48 | 19                      | person:years            | 1981                    |
     49 +-------------------------+-------------------------+-------------------------+
     50 | 36                      | person:locate           | taiwan                  |
     51 +-------------------------+-------------------------+-------------------------+
     52 | 36                      | person:name             | shellon                 |
     53 +-------------------------+-------------------------+-------------------------+
     54 | 36                      | person:years            | 1981                    |
     55 +-------------------------+-------------------------+-------------------------+
     56 3 row(s) in set. (0.04 sec)
    5657 */
    57 
    58 
    59 
    6058
    6159package tw.org.nchc.code;
    … …  
    7775import org.apache.hadoop.mapred.lib.IdentityReducer;
    7876
    79 
    8077public class HBaseRecord {
    8178
    8279  /* Denify parameter */
    8380  // one column family: person; three column qualifier: name,locate,years
    84   static private String  baseId1 ="person:name";
    85   static private String  baseId2 ="person:locate";
    86   static private String  baseId3 ="person:years";
    87   //split character
     81  static private String baseId1 = "person:name";
     82
     83  static private String baseId2 = "person:locate";
     84
     85  static private String baseId3 = "person:years";
     86
     87  // split character
    8888  static private String sp = ":";
     89
    8990  // file path in hadoop file system (not phisical file system)
    9091  String file_path = "/user/waue/t1";
     92
    9193  // Hbase table name
    9294  String table_name = "t1_table";
     95
    9396  // setup MapTask and Reduce Task
    9497  int mapTasks = 1;
     98
    9599  int reduceTasks = 1;
    96  
     100
    97101  private static class ReduceClass extends TableReduce<LongWritable, Text> {
    98102
    99     // Column id is created dymanically, 
     103    // Column id is created dymanically,
    100104    private static final Text col_name = new Text(baseId1);
     105
    101106    private static final Text col_local = new Text(baseId2);
     107
    102108    private static final Text col_year = new Text(baseId3);
    103    
     109
    104110    // this map holds the columns per row
    105     private MapWritable map = new MapWritable(); 
    106    
     111    private MapWritable map = new MapWritable();
     112
    107113    // on this sample, map is nonuse, we use reduce to handle
    108114    public void reduce(LongWritable key, Iterator<Text> values,
    … …  
    110116        throws IOException {
    111117
    112       // values.next().getByte() can get value and transfer to byte form, there is an other way that let decode()
    113       // to substitude getByte()
     118      // values.next().getByte() can get value and transfer to byte form,
     119      // there is an other way that let decode()
     120      // to substitude getByte()
    114121      String stro = new String(values.next().getBytes());
    115122      String str[] = stro.split(sp);
    … …  
    117124      byte b_name[] = str[1].getBytes();
    118125      byte b_year[] = str[2].getBytes();
    119      
     126
    120127      // contents must be ImmutableBytesWritable
    121       ImmutableBytesWritable w_local = new ImmutableBytesWritable( b_local);
    122       ImmutableBytesWritable w_name = new ImmutableBytesWritable( b_name );
    123       ImmutableBytesWritable w_year = new ImmutableBytesWritable( b_year );
     128      ImmutableBytesWritable w_local = new ImmutableBytesWritable(b_local);
     129      ImmutableBytesWritable w_name = new ImmutableBytesWritable(b_name);
     130      ImmutableBytesWritable w_year = new ImmutableBytesWritable(b_year);
    124131
    125132      // populate the current row
    … …  
    141148   */
    142149  public static void main(String[] args) throws IOException {
    143     // which path of input files in Hadoop file system 
    144    
     150    // which path of input files in Hadoop file system
     151
    145152    HBaseRecord setup = new HBaseRecord();
    146153    JobConf conf = new JobConf(HBaseRecord.class);
    147154
    148     //Job name; you can modify to any you like 
     155    // Job name; you can modify to any you like
    149156    conf.setJobName("NCHC_PersonDataBase");
    150157
    151158    // Hbase table name must be correct , in our profile is t1_table
    152159    TableReduce.initJob(setup.table_name, ReduceClass.class, conf);
    153    
     160
    154161    // below are map-reduce profile
    155162    conf.setNumMapTasks(setup.mapTasks);
    156163    conf.setNumReduceTasks(setup.reduceTasks);
    157     conf.setInputPath(new Path(setup.file_path));
     164
     165    // 0.16
     166    // conf.setInputPath(new Path(setup.file_path));
     167    Convert.setInputPath(conf, new Path(setup.file_path));
     168
    158169    conf.setMapperClass(IdentityMapper.class);
    159170    conf.setCombinerClass(IdentityReducer.class);

  • sample/HBaseRecord2.java

    r14 r18  
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/01/2008
     5 * Last Update Date: 07/01/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    5354package tw.org.nchc.code;
    5455
    55 import java.io.FileInputStream;
    5656import java.io.IOException;
    5757import java.util.Iterator;
    … …  
    140140          + "\" has already existed !");
    141141    }
    142     FileInputStream fi = new FileInputStream(setup.file_path);
    143    
    144142   
    145143    JobConf conf = new JobConf(HBaseRecord2.class);
    … …  
    154152    conf.setNumMapTasks(setup.mapTasks);
    155153    conf.setNumReduceTasks(setup.reduceTasks);
    156     conf.setInputPath(new Path(setup.file_path));
     154    // 0.16
     155//    conf.setInputPath(new Path(setup.file_path));
     156    Convert.setInputPath(conf, new Path(setup.file_path));
    157157    conf.setMapperClass(IdentityMapper.class);
    158158    conf.setCombinerClass(IdentityReducer.class);

  • sample/HBaseRecordPro.java

    r17 r18  
    11/**
    2  * Program: HBaseRecord.java
     2 * Program: HBaseRecordPro.java
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/01/2008
     5 * Last Update Date: 07/02/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    227228    conf.setNumMapTasks(mapTasks);
    228229    conf.setNumReduceTasks(reduceTasks);
    229     conf.setInputPath(text_path);
     230    // 0.16
     231//    conf.setInputPath(text_path);
     232    Convert.setInputPath(conf, text_path);
     233   
    230234    conf.setMapperClass(IdentityMapper.class);
    231235    conf.setCombinerClass(IdentityReducer.class);
    … …  
    235239   
    236240    // delete tmp file
    237     FileSystem.get(conf).delete(text_path);
     241    // 0.16
     242//    FileSystem.get(conf).delete(text_path);
     243    FileSystem.get(conf).delete(text_path,true);
     244   
    238245    setup.deleteFile(conf_tmp);
    239246  }

  • sample/WordCount.java

    r9 r18  
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/13/2008
     5 * Last Update Date: 07/02/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    4041import org.apache.hadoop.mapred.Reporter;
    4142
    42 
    4343public class WordCount {
    4444  private String filepath;
     45
    4546  private String outputPath;
    46  
    47   public WordCount(){
     47
     48  public WordCount() {
    4849    filepath = "/user/waue/input/";
    4950    outputPath = "counts1";
    5051  }
    51   public WordCount(String path,String output){
     52
     53  public WordCount(String path, String output) {
    5254    filepath = path;
    5355    outputPath = output;
    5456  }
     57
    5558  // mapper: emits (token, 1) for every word occurrence
    5659  private static class MapClass extends MapReduceBase implements
    … …  
    5962    // reuse objects to save overhead of object creation
    6063    private final static IntWritable one = new IntWritable(1);
     64
    6165    private Text word = new Text();
    6266
    … …  
    9397  }
    9498
    95  
    9699  /**
    97100   * Runs the demo.
    … …  
    99102  public static void main(String[] args) throws IOException {
    100103    WordCount wc = new WordCount();
    101    
     104
    102105    int mapTasks = 1;
    103106    int reduceTasks = 1;
    … …  
    107110    conf.setNumMapTasks(mapTasks);
    108111    conf.setNumReduceTasks(reduceTasks);
    109 
    110     conf.setInputPath(new Path(wc.filepath));
     112    // 0.16
     113    // conf.setInputPath(new Path(wc.filepath));
     114    Convert.setInputPath(conf, new Path(wc.filepath));
    111115    conf.setOutputKeyClass(Text.class);
    112116    conf.setOutputValueClass(IntWritable.class);
    113     conf.setOutputPath(new Path(wc.outputPath));
     117    // 0.16
     118    // conf.setOutputPath(new Path(wc.outputPath));
     119    Convert.setOutputPath(conf, new Path(wc.outputPath));
    114120
    115121    conf.setMapperClass(MapClass.class);
    116122    conf.setCombinerClass(ReduceClass.class);
    117123    conf.setReducerClass(ReduceClass.class);
    118    
     124
    119125    // Delete the output directory if it exists already
    120126    Path outputDir = new Path(wc.outputPath);
    121     FileSystem.get(conf).delete(outputDir);
     127    // 0.16
     128    FileSystem.get(conf).delete(outputDir,true);
    122129
    123130    JobClient.runJob(conf);

  • sample/WordCountFromHBase.java

    r9 r18  
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/13/2008
     5 * Last Update Date: 07/02/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    2526import java.util.Iterator;
    2627import java.util.StringTokenizer;
    27 import java.io.FileOutputStream;
    28 import java.io.File;
    29 import java.io.RandomAccessFile;
     28
    3029import org.apache.hadoop.fs.FileSystem;
    3130import org.apache.hadoop.fs.Path;
    … …  
    169168    // input is Hbase format => TableInputFormat
    170169    conf.setInputFormat(TableInputFormat.class);
    171     conf.setOutputPath(new Path(outputPath));
     170    // 0.16
     171//    conf.setOutputPath(new Path(outputPath));
     172    Convert.setOutputPath(conf, new Path(outputPath));
    172173//     delete the old path with the same name
    173     FileSystem.get(conf).delete(new Path(outputPath));
     174    FileSystem.get(conf).delete(new Path(outputPath),true);
    174175    JobClient.runJob(conf);
    175176  }

  • sample/WordCountIntoHBase.java

    r8 r18  
    33 * Editor: Waue Chen
    44 * From :  NCHC. Taiwn
    5  * Last Update Date: 06/10/2008
     5 * Last Update Date: 07/02/2008
     6 * Upgrade to 0.17
    67 */
    78
    … …  
    4344
    4445  /* setup parameters */
    45   // $Input_Path. Please make sure the path is correct and contains input files
     46  // $Input_Path. Please make sure the path is correct and contains input
     47  // files
    4648  static final String Input_Path = "/user/waue/simple";
     49
    4750  // Hbase table name, the program will create it
    4851  static final String Table_Name = "word_count5";
     52
    4953  // column name, the program will create it
    50   static final String colstr = "word:text" ;
    51  
     54  static final String colstr = "word:text";
     55
    5256  // constructor
    5357  private WordCountIntoHBase() {
    … …  
    5761    // set (column_family:column_qualify)
    5862    private static final Text col = new Text(WordCountIntoHBase.colstr);
     63
    5964    // this map holds the columns per row
    6065    private MapWritable map = new MapWritable();
     66
    6167    public void reduce(LongWritable key, Iterator<Text> values,
    6268        OutputCollector<Text, MapWritable> output, Reporter reporter)
    6369        throws IOException {
    6470      // contents must be ImmutableBytesWritable
    65       ImmutableBytesWritable bytes =
    66         new ImmutableBytesWritable(values.next().getBytes());     
     71      ImmutableBytesWritable bytes = new ImmutableBytesWritable(values
     72          .next().getBytes());
    6773      map.clear();
    68       // write data 
     74      // write data
    6975      map.put(col, bytes);
    7076      // add the row with the key as the row id
    … …  
    7682   * Runs the demo.
    7783   */
    78   public static void main(String[] args) throws IOException { 
     84  public static void main(String[] args) throws IOException {
    7985    // parse colstr to split column family and column qualify
    8086    String tmp[] = colstr.split(":");
    81     String Column_Family = tmp[0]+":";
    82     String CF[] = {Column_Family};
    83     // check whether create table or not , we don't admit \ 
     87    String Column_Family = tmp[0] + ":";
     88    String CF[] = { Column_Family };
     89    // check whether create table or not , we don't admit \
    8490    // the same name but different structure
    85     BuildHTable build_table = new BuildHTable(Table_Name,CF);
     91    BuildHTable build_table = new BuildHTable(Table_Name, CF);
    8692    if (!build_table.checkTableExist(Table_Name)) {
    8793      if (!build_table.createTable()) {
    8894        System.out.println("create table error !");
    8995      }
    90     }else{
    91       System.out.println("Table \"" + Table_Name +"\" has already existed !");
     96    } else {
     97      System.out.println("Table \"" + Table_Name
     98          + "\" has already existed !");
    9299    }
    93100    int mapTasks = 1;
    … …  
    100107    conf.setNumMapTasks(mapTasks);
    101108    conf.setNumReduceTasks(reduceTasks);
    102     conf.setInputPath(new Path(Input_Path));
     109    // 0.16
     110    // conf.setInputPath(new Path(Input_Path));
     111    Convert.setInputPath(conf, new Path(Input_Path));
    103112    conf.setMapperClass(IdentityMapper.class);
    104113    conf.setCombinerClass(IdentityReducer.class);
Note: See TracChangeset for help on using the changeset viewer.