Improved Documentation of Hadoop/MapReduce WebFinder Example

Author: Thomas Weise

hadoop/README.md

@@ -235,8 +235,8 @@ In order to run Hadoop in a pseudo-distributed fashion, we need to enable passwo
 <li>In the terminal, execute <code>ssh localhost</code> to test if you can open a <a href="https://en.wikipedia.org/wiki/Secure&#95;Shell">secure shell</a> connection to your current, local computer <a href="http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SingleCluster.html#Setup&#95;passphraseless&#95;ssh">without needing a password</a>.
 </li>
 <li>It may say something like:
-<pre>ssh: connect to host localhost port 22: Connection refused</pre>.
-If it does say this, then do
+<pre>ssh: connect to host localhost port 22: Connection refused</pre>
+If it does say this (i.e., you did not install the pre-requisites&hellip;), then do
 <pre>sudo apt-get install ssh</pre>
 and it may say something like
 <pre>

hadoop/webFinder/src/main/java/webFinder/WebFinderDriver.java

@@ -14,6 +14,10 @@
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+/**
+ * The driver of the web finder sets up the distributed computation by
+ * defining what the mapper and reducer classes, amongst other things.
+ */
 public class WebFinderDriver extends Configured implements Tool {
 
   public static void main(final String[] args) throws Exception {
@@ -27,6 +31,7 @@ public static void main(final String[] args) throws Exception {
     }
   }
 
+  /** Setting up the computation. */
   @Override
   public int run(final String[] args) throws Exception {
     final Configuration conf;

hadoop/webFinder/src/main/java/webFinder/WebFinderMapper.java

@@ -13,12 +13,22 @@
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.log4j.Logger;
 
-public class WebFinderMapper
-    extends Mapper<LongWritable, Text, Text, Text> {
+/**
+ * This is the Mapper component of the Web Finder example. Its input are
+ * text lines, where each line stands for a website URL. It finds all
+ * resources that are loaded by a given website URL and emits tuples of
+ * kind {@code <resource URL, website URL>}.
+ */
+public class WebFinderMapper extends
+    Mapper<LongWritable, Text, Text, Text> {
 
   /** the logger we use */
   private static Logger LOGGER = Logger.getLogger(WebFinderMapper.class);
 
+  /**
+   * Map tuples of type {@code <line number, website url text>} to tuples
+   * of kind {@code <resource url text, website url text>}.
+   */
   @Override
   protected void map(final LongWritable offset, final Text line,
       final Context context) throws IOException, InterruptedException {
@@ -171,8 +181,8 @@ private static final void __load(final int remainingDepth,
                 error.addSuppressed(error2);
                 error.addSuppressed(error3);
                 if (WebFinderMapper.LOGGER != null) {
-                  WebFinderMapper.LOGGER
-                      .warn("Error while trying to build URL with string '"
+                  WebFinderMapper.LOGGER.warn(
+                      "Error while trying to build URL with string '"
                           + test + "' under load URL '"
                           + loadUrl.toString() + "' for base URL '"
                           + baseUrl.toString() + "'.", error2);
@@ -215,8 +225,8 @@ private static final void __load(final int remainingDepth,
       }
     } catch (final Throwable error) {
       if (WebFinderMapper.LOGGER != null) {
-        WebFinderMapper.LOGGER.warn(
-            "Error while trying to load URL '" + loadUrl + "'.", error);
+        WebFinderMapper.LOGGER.warn("Error while trying to load URL '"
+            + loadUrl + "'.", error);
       }
     }
 
@@ -314,7 +324,7 @@ public static final void main(final String[] args) throws Throwable {
 
   /** the link descriptions */
   static final __LinkDesc[] DESCS = { //
-      new __LinkDesc(false, "<link rel=\"stylesheet\"", "href="), //
+  new __LinkDesc(false, "<link rel=\"stylesheet\"", "href="), //
       new __LinkDesc(false, "<link rel='stylesheet'", "href="), //
       new __LinkDesc(false, "<img", "src="), //
       new __LinkDesc(false, "<script", "src="), //

hadoop/webFinder/src/main/java/webFinder/WebFinderReducer.java

@@ -11,9 +11,23 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Reducer;
 
-public class WebFinderReducer
-    extends Reducer<Text, Text, Text, List<Text>> {
+/**
+ * This is the reducer component of the web finder example. For each key (
+ * {@code resource URL}) of the tuples produced by the mapper, it receives
+ * the list of all values ({@code website URLs}). If such a list contains
+ * more than one unique element, this means that the resource is shared by
+ * multiple websites. This reducer emits tuples of the form
+ * {@code <resource URL, list of website urls>}.
+ */
+public class WebFinderReducer extends
+    Reducer<Text, Text, Text, List<Text>> {
 
+  /**
+   * The actual reduction step: From the tuples of form
+   * {@code <resource URL, iterable of referencing website URLs>}, select
+   * all resources referenced by more than one unique website. For these,
+   * output tuples of the form {@code <resource URL, list of website URLs>}.
+   */
   @Override
   protected void reduce(final Text key, final Iterable<Text> values,
       final Context context) throws IOException, InterruptedException {