Changeset 80

Timestamp:

Jun 9, 2009, 6:08:55 PM (16 years ago)

Author:

waue

Message:

NutchEz ..0.1

Files:

• nutchez-0.1/bin/nutchez-func.sh (modified) (1 diff)
• nutchez-0.1/conf/nutch-site.xml (modified) (1 diff)
• nutchez-0.1/conf/sav/n.urls.txt (modified) (1 diff)
• nutchez-0.1/debian/nutchez.install (modified) (1 diff)
• nutchez-0.1/debian/nutchez.postrm (modified) (1 diff)
• package (added)
• package/nutchez_0.1-1_i386.deb (added)

nutchez-0.1/bin/nutchez-func.sh

@@ -169 +169 @@
   echo_vb "/opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH"
   echo_vb "nutch conf dir = $NUTCH_CONF_DIR"
-  /opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH
+  /opt/nutch/bin/nutch crawl ~/.nutchez/urls -dir ~/.nutchez/search -depth $DEPTH -topN 5000 -threads 1000
 }
 

nutchez-0.1/conf/nutch-site.xml

@@ -66 +66 @@
   </description>
 </property>
-
+<property>
+  <name>db.ignore.external.links</name>
+  <value>false</value>
+  <description>If true, outlinks leading from a page to external hosts
+  will be ignored. This is an effective way to limit the crawl to include
+  only initially injected hosts, without creating complex URLFilters.
+  </description>
+</property>
+<property>
+  <name>file.content.limit</name>
+  <value>1000000</value>
+  <description>The length limit for downloaded content, in bytes.
+  If this value is nonnegative (>=0), content longer than it will be truncated;
+  otherwise, no truncation at all.
+  </description>
+</property>
 </configuration>
 

nutchez-0.1/conf/sav/n.urls.txt

@@ -1 @@
-http://www.nchc.org.tw
+http://www.nchc.org.tw/tw/
 http://www.hadoop.tw

nutchez-0.1/debian/nutchez.install

@@ -1 +1 @@
 conf/*    etc/nutch
 bin   opt/nutch
-bin/nutchez*  usr/local/sbin
+bin/nutchez*  usr/local/bin
 lib   opt/nutch
 webapps   opt/nutch

nutchez-0.1/debian/nutchez.postrm

@@ -14 +14 @@
   if [ -d $i ];then
     echo "delete this dir :  $i"
-    rm -ir $i
+    rm -r $i
   fi
 done