Sqoop Import HDFS

Sqoop Import HDFS

Sqoop Import 应用场景——密码访问

明码访问

sqoop list-databases \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

--password 111111

交互式密码

sqoop list-databases \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

-P

文件授权密码

sqoop list-databases \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

--password-file /user/hadoop/.password

Sqoop Import 应用场景——导入全表

不指定目录

sqoop import \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

--password-file /user/hadoop/.password \

--table dwq-user

不指定目录

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user

指定目录

sqoop import \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

--password-file /user/hadoop/.password \

--table dwq-user \

--target-dir /sqoop/test/dwq_user

目录已存在

sqoop import \

--connect jdbc:mysql://192.168.x.200/dwqdb_test \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir

Sqoop Import 应用场景——控制并行度

控制并行度

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1

Sqoop Import 应用场景——控制字段分隔符

控制字段分隔符

sqoop import \

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@"

Sqoop Import 应用场景——增量导入

手动增量导入

sqoop import \

--connect 'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

-m 1 \

--fields-terminated-by "@" \

--append \

--check-column 'id' \

--incremental append \

--last-value 3

自动增量导入

sqoop job \

--create job_import_dwq_user \

--connect 'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

-m 1 \

--fields-terminated-by "@" \

--append \

--check-column 'id' \

--incremental append \

--last-value 6

sqoop job --exec job_import_dwq_user

Sqoop Import 应用场景——启动压缩

启动压缩

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@" \

-z

Sqoop Import 应用场景——导入空值处理

导入空值处理

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@" \

--null-non-string "###" \

--null-string "###"

Sqoop Import 应用场景——导入部分数据

使用–columns

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--columns id,name \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@" \

--null-non-string "###" \

--null-string "###"

使用–where

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--table dwq_user \

--where "sex='female'" \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@" \

--null-non-string "###" \

--null-string "###"

使用–query

sqoop import \

--connect

'jdbc:mysql://192.168.x.200/dwqdb_test?useUnicode=true&characterEncoding=utf-8' \

--username root \

--password-file /user/hadoop/.password \

--query "select * from dwq_user where age<=30 and \$CONDITIONS" \

--target-dir /sqoop/test/dwq_user \

--delete-target-dir \

-m 1 \

--fields-terminated-by "@" \

--null-non-string "###" \

--null-string "###"