[HADOOP] ArrayWritable을 통해 반복-NoSuchMethodException

ArrayWritable을 통해 반복-NoSuchMethodException

방금 MapReduce로 작업을 시작했으며 Google을 통해 답변을 할 수 없었던 이상한 버그가 발생했습니다. ArrayWritable을 사용하여 기본 프로그램을 만들고 있지만 실행하면 Reduce 중에 다음 오류가 발생합니다.

java.lang.RuntimeException:
java.lang.NoSuchMethodException:org.apache.hadoop.io.ArrayWritable.<init>()
at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:115)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:62)
at org.apache.hadoop.io.serializer.WritableSerialization$WritableDeserializer.deserialize(WritableSerialization.java:40)
at org.apache.hadoop.mapred.Task$ValuesIterator.readNextValue(Task.java:1276)
at org.apache.hadoop.mapred.Task$ValuesIterator.next(Task.java:1214)
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.moveToNext(ReduceTask.java:250)
at org.apache.hadoop.mapred.ReduceTask$ReduceValuesIterator.next(ReduceTask.java:246)
at PageRank$Reduce.reduce(Unknown Source)
at PageRank$Reduce.reduce(Unknown Source)
at org.apache.hadoop.mapred.ReduceTask.runOldReducer(ReduceTask.java:522)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:421)
at org.apache.hadoop.mapred.Child$4.run(Child.java:255)

하둡 1.2.1을 사용하고 있습니다. 내 코드는 다음과 같습니다.

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.join.*;
import java.io.IOException;
import java.util.Iterator;

public class TempClass {

  public static class MapClass extends MapReduceBase
  implements Mapper<LongWritable, Text, Text, ArrayWritable> {
    public void map(LongWritable key, Text value,
        OutputCollector<Text, ArrayWritable> output,
        Reporter reporter) throws IOException {

      String[] arr_str = new String[]{"a","b","c"};
      for(int i=0; i<3; i++)
        output.collect(new Text("my_key"), new ArrayWritable(arr_str));
    }
  }    

  public static class Reduce extends MapReduceBase
  implements Reducer<Text, ArrayWritable, Text, ArrayWritable> {

    public void reduce(Text key, Iterator<ArrayWritable> values,
        OutputCollector<Text, ArrayWritable> output,
        Reporter reporter) throws IOException {

      ArrayWritable tmp;

      while(values.hasNext()){
          tmp = values.next();
          output.collect(key, tmp);
      }
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();

    JobConf job = new JobConf(conf, TempClass.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(ArrayWritable.class);
    job.setOutputFormat(TextOutputFormat.class);
    job.setInputFormat(TextInputFormat.class);

    job.setMapperClass(MapClass.class);
    job.setReducerClass(Reduce.class);

    FileInputFormat.setInputPaths( job, new Path( args[0] ) );
    FileOutputFormat.setOutputPath( job, new Path( args[1] ) );

    job.setJobName( "TempClass" );

    JobClient.runJob(job);
  }
}

아래 줄에 댓글을 달면 (Reduce Class) :

      //while(values.hasNext()){
      //    tmp = values.next();
          output.collect(key, tmp);
      //}

모든 게 잘 될거야. 당신은 어떤 아이디어가 있습니까?

해결법

1. ==============================

   1.다음은 ArrayWritable 문서에서 발췌 한 것입니다. 일반적으로 Writable에는 매개 변수가없는 생성자가 있어야합니다.

   다음은 ArrayWritable 문서에서 발췌 한 것입니다. 일반적으로 Writable에는 매개 변수가없는 생성자가 있어야합니다.

   방금 코드를 다음과 같이 수정했습니다.

   import java.io.IOException;
   import java.util.Iterator;
   
   import org.apache.hadoop.conf.Configuration;
   import org.apache.hadoop.fs.Path;
   import org.apache.hadoop.io.ArrayWritable;
   import org.apache.hadoop.io.LongWritable;
   import org.apache.hadoop.io.Text;
   import org.apache.hadoop.mapred.FileInputFormat;
   import org.apache.hadoop.mapred.FileOutputFormat;
   import org.apache.hadoop.mapred.JobClient;
   import org.apache.hadoop.mapred.JobConf;
   import org.apache.hadoop.mapred.MapReduceBase;
   import org.apache.hadoop.mapred.Mapper;
   import org.apache.hadoop.mapred.OutputCollector;
   import org.apache.hadoop.mapred.Reducer;
   import org.apache.hadoop.mapred.Reporter;
   import org.apache.hadoop.mapred.TextInputFormat;
   import org.apache.hadoop.mapred.TextOutputFormat;
   
   public class TempClass {
   
       public static class TextArrayWritable extends ArrayWritable {
           public TextArrayWritable() {
               super(Text.class);
           }
   
           public TextArrayWritable(String[] strings) {
               super(Text.class);
               Text[] texts = new Text[strings.length];
               for (int i = 0; i < strings.length; i++) {
                   texts[i] = new Text(strings[i]);
               }
               set(texts);
           }
       }
   
       public static class MapClass extends MapReduceBase implements
               Mapper<LongWritable, Text, Text, ArrayWritable> {
           public void map(LongWritable key, Text value,
                   OutputCollector<Text, ArrayWritable> output, Reporter reporter)
                   throws IOException {
   
               String[] arr_str = new String[] {
                       "a", "b", "c" };
               for (int i = 0; i < 3; i++)
                   output.collect(new Text("my_key"), new TextArrayWritable(
                           arr_str));
           }
       }
   
       public static class Reduce extends MapReduceBase implements
               Reducer<Text, TextArrayWritable, Text, TextArrayWritable> {
   
           public void reduce(Text key, Iterator<TextArrayWritable> values,
                   OutputCollector<Text, TextArrayWritable> output,
                   Reporter reporter) throws IOException {
   
               TextArrayWritable tmp;
   
               while (values.hasNext()) {
                   tmp = values.next();
                   output.collect(key, tmp);
               }
           }
       }
   
       public static void main(String[] args) throws Exception {
           Configuration conf = new Configuration();
   
           JobConf job = new JobConf(conf, TempClass.class);
   
           job.setOutputKeyClass(Text.class);
           job.setOutputValueClass(TextArrayWritable.class);
           job.setOutputFormat(TextOutputFormat.class);
           job.setInputFormat(TextInputFormat.class);
   
           job.setMapperClass(MapClass.class);
           job.setReducerClass(Reduce.class);
   
           FileInputFormat.setInputPaths(job, new Path(args[0]));
           FileOutputFormat.setOutputPath(job, new Path(args[1]));
   
           job.setJobName("TempClass");
   
           JobClient.runJob(job);
       }
   }
   

from https://stackoverflow.com/questions/24274668/iterate-through-arraywritable-nosuchmethodexception by cc-by-sa and MIT license