Hadoop快速入门——入门考试(伪分布式60+编码25+执行jar15)

tar -zxvf jdk-8u112-linux-x64.tar.gz
tar -zxvf hadoop-2.7.3.tar.gz

mv jdk1.8.0_112 jdk
mv hadoop-2.7.3 hadoop

vi /etc/profile.d/hadoop-eco.sh

JAVA_HOME=/opt/jdk
PATH=$JAVA_HOME/bin:$PATH
HADOOP_HOME=/opt/hadoop
PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

ssh-keygen -t rsa

ssh-copy-id -i root@localhost

ssh root@localhost

exit

mkdir -p /opt/hadoop-record/name
mkdir -p /opt/hadoop-record/secondary
mkdir -p /opt/hadoop-record/data
mkdir -p /opt/hadoop-record/tmp
ls /opt/hadoop-record/

source /etc/profile.d/hadoop-eco.sh
java -version

hdfs namenode -format
start-all.sh
jps

systemctl stop firewalld

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.3</version>
        </dependency>
    </dependencies>

package com.item.test;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class MapAction extends Mapper<LongWritable, Text, Text, LongWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        char[] split = value.toString().toCharArray();
        for (char s : split) {
            context.write(new Text(s+""), new LongWritable(1));
        }
    }
}

package com.item.test;


import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class ReduceAction extends Reducer<Text, LongWritable, Text, LongWritable> {
    @Override
    protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
        long count = 0;
        for (LongWritable value : values) {
            count += value.get();
        }
        context.write(key, new LongWritable(count));
    }
}

package com.item.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class Action {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(Action.class);
        job.setMapperClass(MapAction.class);
        job.setReducerClass(ReduceAction.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);

        FileInputFormat.setInputPaths(job,new Path("/info/info.txt"));
        FileOutputFormat.setOutputPath(job,new Path("/infos"));

        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);
    }
}

cd /opt/hadoop/share/hadoop/mapreduce
echo "Accept that this is your starting point.Instead of placing judgements on it,see the real,positive value that’s already yours.You cannot change where your past priorities and choices have brought you.Yet you can make use of the wisdom you’ve earned to create a future that fulfils your most treasured dreams.
> 相信此刻就是你的起点。无需再判断，看到你所具备的真正意义上的价值。你过去认为重要的事和以前做过的选择给你带来了什么都是无法改变 的。然而你可以充分利用你的智慧去创造未来，实现你最珍爱的梦想。" >info.txt
hadoop fs -mkdir /info
hadoop fs -put info.txt /info/info.txt
hadoop fs -cat /info/info.txt

hadoop jar demo.jar com/item/test/Action /info/info.txt /infos
hadoop fs -cat /infos/part-r-00000