环境与命令速查

SSH(Secure Shell)是可以在应用程序中提供安全通信的一个协议，通过SSH可以安全地进行网络数据传输，它的主要原理就是利用非对称加密体系，对所有待传输的数据进行加密，保证数据在传输时不被恶意破坏、泄露或者篡改。

[hadoop@hadoop1 app]$ vi ~/.bashrc
JAVA_HOME=/home/hadoop/app/jdk
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH

[hadoop@hadoop1 app]$ source ~/.bashrc
[hadoop@hadoop1 app]$ echo $JAVA_HOME

[hadoop@hadoop1 hadoop]$ vi hadoop-env.sh
export JAVA_HOME=/home/hadoop/app/jdk

[hadoop@hadoop1 hadoop]$ vi ~/.bashrc
JAVA_HOME=/home/hadoop/app/jdk
HADOOP_HOME=/home/hadoop/app/hadoop
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH HADOOP_HOME

[hadoop@hadoop1 hadoop]$ source ~/.bashrc
[hadoop@hadoop1 hadoop]$ hadoop version
Hadoop 3.3.1

vi /etc/chrony.conf

systemctl restart chronyd

systemctl enable chronyd

chronyc makestep

<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLAS

[hadoop@hadoop1 conf]$ vi ~/.bashrc

JAVA_HOME=/home/hadoop/app/jdk
HADOOP_HOME=/home/hadoop/app/hadoop
HIVE_HOME=/home/hadoop/app/hive
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:/home/hadoop/tools:$HIVE_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH HADOOP_HOME HIVE_HOME

[hadoop@hadoop1 conf]$ vi ~/.bashrc
export SQOOP_HOME=/home/hadoop/app/sqoop

[hadoop@hadoop1 conf]$ source ~/.bashrc

[hadoop@hadoop1 hadoop]$ vi core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://hadoop1:9000</value>

</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/data/tmp</value>

hdfs-site.xml文件主要配置跟HDFS相关的属性，具体配置内容如下所示。
[hadoop@hadoop1 hadoop]$ vi hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>/home/hadoop/data/dfs/name</value>

</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/hadoop/data/dfs/data</value>

</property>
<property>
<name>dfs.replication</name>
<value>1</value>

</property>
<property>
<name>dfs.permissions</name>
<value>false</value>

[hadoop@hadoop1 hadoop]$ vi mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>

</property>
<property>
<name>yarn.app.mapreduce.am.env</name>
<value>HADOOP_MAPRED_HOME=/home/hadoop/app/hadoop</value>

</property>
<property>
<name>mapreduce.map.env</name>
<value>HADOOP_MAPRED_HOME=/home/hadoop/app/hadoop</value>

</property>
<property>
<name>mapreduce.reduce.env</name>
<value>HADOOP_MAPRED_HOME=/home/hadoop/app/hadoop</value>

[hadoop@hadoop1 hadoop]$ vi yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>

[hadoop@hadoop1 hadoop]$ vi workers

[hadoop@hadoop1 hadoop]$ bin/hdfs namenode -format

[hadoop@hadoop1 hadoop]$ vi core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://mycluster</value>
</property>

<property>
<name>hadoop.tmp.dir</name>
<value>/home/hadoop/data/tmp</value>
</property>

[hadoop@hadoop1 hadoop]$ vi hdfs-site.xml
<configuration>

<property>
<name>dfs.namenode.name.dir</name>
<value>file:/home/hadoop/data/dfs/name</value>
</property>

<property>
<name>dfs.datanode.data.dir</name>
<value>file:/home/hadoop/data/dfs/data</value>
</property>

<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/hadoop/data/journaldata/jn</value>
</property>

<property>
<name>dfs.nameservices</name>
<value>mycluster</value>
</property>

<property>
<name>dfs.ha.namenodes.mycluster</name>
<value>nn1,nn2,nn3</value>
</property>

<property>
<name>dfs.namenode.rpc-address.mycluster.nn1</name>
<value>hadoop1:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn2</name>
<value>hadoop2:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.mycluster.nn3</name>
<value>hadoop3:8020</value>
</property>

<property>
<name>dfs.namenode.http-address.mycluster.nn1</name>
<value>hadoop1:9870</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn2</name>
<value>hadoop2:9870</value>
</property>
<property>
<name>dfs.namenode.http-address.mycluster.nn3</name>
<value>hadoop3:9870</value>
</property>

<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop1:8485;hadoop2:8485;hadoop3:8485/mycluster</value>
</property>

<property>
<name>dfs.client.failover.proxy.provider.mycluster</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>

<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>

<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadoop/.ssh/id_rsa</value>
</property>

<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.permissions.enabled</name>
<value>false</value>
</property>
</configuration>

[hadoop@hadoop1 hadoop]$sbin/hadoop-daemon.sh start journalnode
[hadoop@hadoop2 hadoop]$sbin/hadoop-daemon.sh start journalnode
[hadoop@hadoop3 hadoop]$sbin/hadoop-daemon.sh start journalnode

[hadoop@hadoop1 hadoop]$ runRemoteCmd.sh '/home/hadoop/app/hadoop/sbin/hadoop-daemon.sh start journalnode' all

[hadoop@hadoop2 hadoop]$ bin/hdfs namenode -bootstrapStandby
[hadoop@hadoop3 hadoop]$ bin/hdfs namenode -bootstrapStandby

[hadoop@hadoop1 hadoop]$sbin/hadoop-daemon.sh stop journalnode
[hadoop@hadoop2 hadoop]$ sbin/hadoop-daemon.sh stop journalnode
[hadoop@hadoop3 hadoop]$ sbin/hadoop-daemon.sh stop journalnode

[hadoop@hadoop1 hadoop]$ sbin/start-dfs.sh

[hadoop@hadoop1 hadoop]$ vi yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>

<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>cluster-yarn1</value>
</property>

<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2,rm3</value>
</property>

<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hadoop1</value>
</property>

<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>hadoop1:8088</value>
</property>

<property>
<name>yarn.resourcemanager.address.rm1</name>
<value>hadoop1:8032</value>
</property>

[hadoop@hadoop1 app]$ tar -zxvf apache-zookeeper-3.8.4-bin.tar.gz

[hadoop@hadoop1 app]$ ln -s apache-zookeeper-3.8.4-bin zookeeper

[hadoop@hadoop1 app]$ cd zookeeper
[hadoop@hadoop1 zookeeper]$ cd conf/
[hadoop@hadoop1 conf]$ ls
configuration.xsl logback.xml zoo_sample.cfg
[hadoop@hadoop1 conf]$ cp zoo_sample.cfg zoo.cfg
[hadoop@hadoop1 conf]$ vi zoo.cfg

dataDir=/home/hadoop/data/zookeeper/zkdata

dataLogDir=/home/hadoop/data/zookeeper/zkdatalog

clientPort=2181

server.1=hadoop1:2888:3888
server.2=hadoop2:2888:3888
server.3=hadoop3:2888:3888

[hadoop@hadoop1 app]$scp -r zookeeper-3.4.6 hadoop@hadoop2:/home/hadoop/app/
[hadoop@hadoop1 app]$scp -r zookeeper-3.4.6 hadoop@hadoop3:/home/hadoop/app/

[hadoop@hadoop1 app]$ deploy.sh apache-zookeeper-3.8.4-bin /home/hadoop/app/ slave

[hadoop@hadoop2 app]$ ln -s apache-zookeeper-3.8.4-bin zookeeper
[hadoop@hadoop3 app]$ ln -s apache-zookeeper-3.8.4-bin zookeeper

[hadoop@hadoop1 app]$mkdir -p /home/hadoop/data/zookeeper/zkdata
[hadoop@hadoop2 app]$mkdir -p /home/hadoop/data/zookeeper/zkdata
[hadoop@hadoop3 app]$mkdir -p /home/hadoop/data/zookeeper/zkdata

[hadoop@hadoop1 ~]$ runRemoteCmd.sh 'mkdir -p /home/hadoop/data/zookeeper/zkdata' all

[hadoop@hadoop1 app] mkdir -p /home/hadoop/data/zookeeper/zkdatalog
[hadoop@hadoop2 app] mkdir -p /home/hadoop/data/zookeeper/zkdatalog
[hadoop@hadoop3 app] mkdir -p /home/hadoop/data/zookeeper/zkdatalog

[hadoop@hadoop1 ~]$ runRemoteCmd.sh 'mkdir -p /home/hadoop/data/zookeeper/zkdatalog' all

[hadoop@hadoop1 zkdata]$ touch myid
[hadoop@hadoop1 zkdata]$ echo 1 > myid

[hadoop@hadoop2 zkdata]$ touch myid
[hadoop@hadoop2 zkdata]$ echo 2 > myid

[hadoop@hadoop3 zkdata]$ touch myid
[hadoop@hadoop3 zkdata]$ echo 3 > myid

[hadoop@hadoop1 zookeeper]$ bin/zkServer.sh start
[hadoop@hadoop2 zookeeper]$ bin/zkServer.sh start
[hadoop@hadoop3 zookeeper]$ bin/zkServer.sh start

[hadoop@hadoop1 ~]$ runRemoteCmd.sh '/home/hadoop/app/zookeeper/bin/zkServer.sh start' all

[hadoop@hadoop:1 zookeeper]$ bin/zkServer.sh status
[hadoop@hadoop2 zookeeper]$ bin/zkServer.sh status
[hadoop@hadoop3 zookeeper]$ bin/zkServer.sh status

[hadoop@hadoop1 ~]$ runRemoteCmd.sh '/home/hadoop/app/zookeeper/bin/zkServer.sh status' all

<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop1:2181,hadoop2:2181,hadoop3:2181</value>
</property>

[hadoop@hadoop1 hadoop]$ runRemoteCmd.sh '/home/hadoop/app/zookeeper/bin/zkServer.sh start ' all

[hadoop@hadoop1 conf]$ cp sqoop-env-template.sh sqoop-env.sh
[hadoop@hadoop1 conf]$ vi sqoop-env.sh
export HADOOP_COMMON_HOME=/home/hadoop/app/hadoop
export HADOOP_MAPRED_HOME=/home/hadoop/app/hadoop
export HIVE_HOME=/home/hadoop/app/hive
export ZOOCFGDIR=/home/hadoop/app/zookeeper
export HCAT_HOME=/home/hadoop/app/hive/hcatalog

mysql>alter user'root'@'localhost' identified with mysql_native_password by 'root';

mysql>create user 'hive'@'hadoop1' identified by 'hive';
mysql>create user 'hive'@'%' identified by 'hive';

mysql>GRANT ALL PRIVILEGES ON *.* TO 'hive'@'hadoop1' WITH GRANT OPTION;

mysql>GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%' WITH GRANT OPTION;

mysql> flush privileges;

mysql> create database hive;

<property>
<name>javax.jdo.option.ConnectionDriverName</name>
<value>com.mysql.jdbc.Driver</value>
</property>i

<property>
<name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://hadoop1:3306/hive?characterEncoding=UTF-8</value>
</property>

[hadoop@hadoop1 sqoop]$ bin/sqoop list-databases --connect jdbc:mysql://hadoop1 --username hive --password hive

mysql>source /home/hadoop/shell/sql/nev.sql

[hadoop@hadoop1 sqoop]$bin/sqoop import \
--connect jdbc:mysql://191.168.2410.161:3306/nev \
--username hive \
--password hive \
--query "select

[hadoop@hadoop1 sqoop]$bin/sqoop import \
--connect jdbc:mysql://191.168.2410.161:3306/nev \
--username hive \
--password hive \
--query "select id,from_url,landing_page_url,url_title,platform_description,other_params,history from web_chat_text_ems_2019_07 where 1=1 and \$CONDITIONS" \
--hcatalog-database hnsoft_ods \
--hcatalog-table web_chat_text_ems_ods \

bin/sqoop export \
--connect "jdbc:mysql://191.168.249.161:3306/nev?useUnicode=true&characterEncoding=utf-8" \
--username hive \
--password hive \
--table visit_dws \
--hcatalog-database hnsoft_dws \
--hcatalog-table visit_dws

bin/sqoop export \
--connect "jdbc:mysql://191.168.249.161:3306/nev?useUnicode=true&characterEncoding=utf-8" \
--username hive \
--password hive \
--table consult_dws \
--hcatalog-database hnsoft_dws \
--hcatalog-table consult_dws

[hadoop@hadoop1 conf]$ cp hive-default.xml.template hive-site.xml

[hadoop@hadoop1 conf]$ vi hive-site.xml

[hadoop@hadoop1 conf]$ vi hive-site.xml
<property>
<name>hive.querylog.location</name>
<value>/home/hadoop/app/hive/iotmp</value>
</property>
<property>
<name>hive.exec.local.scratchdir</name>
<value>/home/hadoop/app/hive/iotmp</value>
</property>
<property>
<name>hive.downloaded.resources.dir</name>
<value>/home/hadoop/app/hive/iotmp</value>
</property>

[hadoop@hadoop1 hive]$ bin/schematool -dbType mysql -initSchema

[hadoop@hadoop102 conf]$ vim hive-site.xml

[hadoop@hadoop1 hive]$ bin/hive --service hiveserver2

[hanxiao@hadoop102 conf]$ cp hive-env.sh.template hive-env.sh

[hanxiao@hadoop102 conf]$ vim hive-env.sh

[root@localhost ~]# tar -zxvf dataease-v1.18.19-offline.tar.gz

[root@localhost ~]# cd dataease-v1.18.19-offline
[root@localhost dataease-v1.18.19-offline]# bash install.sh

[root@hadoop1 ~]# yum install tar -y

[hadoop@hadoop1 app]$ ls

[hadoop@hadoop1 app]$ tar -zxvf jdk-8u141-linux-x64.tar.gz

[hadoop@hadoop1 app]$ ln -s jdk1.8.0_141 jdk

[hadoop@hadoop1 app]$ java -version
java version "1.8.0_141"
Java(TM) SE Runtime Environment (build 1.8.0_141-b15)
Java HotSpot(TM) 64-Bit Server VM (build 25.141-b15, mixed mode)

[hadoop@hadoop1 app]$ ls
hadoop-3.3.1.tar.gz

[hadoop@hadoop1 app]$ tar -zxvf hadoop-3.3.1.tar.gz

[hadoop@hadoop1 app]$ ln -s hadoop-3.3.1 hadoop

</property>
</configuration>

[hadoop@hadoop1 hadoop]$ mkdir -p /home/hadoop/data/tmp
[hadoop@hadoop1 hadoop]$ mkdir -p /home/hadoop/data/dfs/name
[hadoop@hadoop1 hadoop]$ mkdir -p /home/hadoop/data/dfs/data

[hadoop@hadoop1 hadoop]$ sbin/start-all.sh

[hadoop@hadoop1 ~]$ jps

[hadoop@hadoop1 hadoop]$ bin/hdfs dfs -ls /

[hadoop@hadoop1 hadoop]$ bin/hdfs dfs -mkdir /test

[hadoop@hadoop1 hadoop]$ vi words.log
hadoop hadoop hadoop
spark spark spark
flink flink flink

[hadoop@hadoop1 hadoop]$ bin/hdfs dfs -put words.log /test

[hadoop@hadoop1 hadoop]$ bin/yarn jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.1.jar wordcount /test/words.log /test/out

[hadoop@hadoop1 hadoop]$ bin/hdfs dfs -cat /test/out/*
flink 3
hadoop 3
spark 3

# pool 2.centos.pool.ntp.org iburst

server ntp.aliyun.com iburst

[hadoop@hadoop2 ~]$ cat ~/.ssh/id_rsa.pub | ssh hadoop@hadoop1 'cat >> ~/.ssh/authorized_keys'

[hadoop@hadoop3 ~]$ cat ~/.ssh/id_rsa.pub | ssh hadoop@hadoop1 'cat >> ~/.ssh/authorized_keys'

[hadoop@hadoop1 .ssh]$ scp -r authorized_keys hadoop@hadoop2:~/.ssh

[hadoop@hadoop1 .ssh]$ scp -r authorized_keys hadoop@hadoop3:~/.ssh/