FlinkCDC Mysql To Starrocks 

FlinkCDC Mysql To Starrocks 

一、开发实时同步脚本
#智慧园区表--开发实时同步脚本

mysql-to-starrocks_route_db_evoc9_person_ps_person.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: db_evoc9_person\d+.ps_person

   server-id: 6500-6503

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: db_evoc9_person\d+.ps_person

     sink-table: bigdata_bi.ods_db_evoc9_person_ps_person

 

pipeline:

   name: route_db_evoc9_person_ps_person(人员信息表-分库分表)

   parallelism: 3
 

 
mysql-to-starrocks_route_db_evoc_deviceinfo_alarm_info.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: db_evoc_deviceinfo\d+.alarm_info

   server-id: 5501

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: db_evoc_deviceinfo\d+.alarm_info

     sink-table: bigdata_bi.ods_db_evoc_deviceinfo_alarm_info

 

pipeline:

   name: route_db_evoc_deviceinfo_alarm_info(报警信息表-分库分表)

   parallelism: 2
 

 
mysql-to-starrocks_route_db_evoc_deviceinfo_device_info.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: db_evoc_deviceinfo\d+.device_info

   server-id: 5506

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: db_evoc_deviceinfo\d+.device_info

     sink-table: bigdata_bi.ods_db_evoc_deviceinfo_device_info

pipeline:

   name: route_db_evoc_deviceinfo_device_info(设备信息表-分库分表)

   parallelism: 2
 

 
mysql-to-starrocks_route_db_evoc_deviceinfo_device_type.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: db_evoc_deviceinfo\d+.device_type

   server-id: 5609

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial
 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

route:

   - source-table: db_evoc_deviceinfo\d+.device_type

     sink-table: bigdata_bi.ods_db_evoc_deviceinfo_device_type

pipeline:

   name: route_db_evoc_deviceinfo_device_type(设备类型表-分库分表)

   parallelism: 1
 

 
mysql-to-starrocks_route_devicex_app_device_info.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: devicex_app.device_info

   server-id: 5502

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true
 

route:

   - source-table: devicex_app.device_info

     sink-table: bigdata_bi.ods_devicex_app_device_info
 

pipeline:

   name: route_devicex_app_device_info(设备机房信息表)

   parallelism: 1

 
mysql-to-starrocks_route_evoc9_message_ms_alarmlevel.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: evoc9_message.ms_alarmlevel

   server-id: 5503

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial
 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

route:

   - source-table: evoc9_message.ms_alarmlevel

     sink-table: bigdata_bi.ods_evoc9_message_ms_alarmlevel

pipeline:

   name: route_evoc9_message_ms_alarmlevel(报警等级表)

   parallelism: 2

 
mysql-to-starrocks_route_evoc9_person_ps_personvisit.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: db_evoc9_person\d+.ps_personvisit

   server-id: 5700-5705

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

   #scan.startup.mode: latest-offset

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: db_evoc9_person\d+.ps_personvisit

     sink-table: bigdata_bi.ods_db_evoc9_person_ps_personvisit

pipeline:

   name: route_evoc9_person_ps_personvisit(人员出入信息表-分库分表)

   parallelism: 5
 

 
mysql-to-starrocks_route_evoc_platform.yaml

source:

   type: mysql

   hostname: 172.16.5.11

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: evoc_platform.\.*

   server-id: 5801-5802

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial
 

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: evoc_platform.pf_park

     sink-table: bigdata_bi.ods_evoc_platform_pf_park

   - source-table: evoc_platform.pf_building

     sink-table: bigdata_bi.ods_evoc_platform_pf_building

   - source-table: evoc_platform.pf_company

     sink-table: bigdata_bi.ods_evoc_platform_pf_company
 

pipeline:

   name: route_evoc_platform(楼栋-公司-园区信息表)

   parallelism: 2
 

 
mysql-to-starrocks_route_positioning.yaml

source:

   type: mysql

   hostname: 172.16.5.15

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: positioning.\.*

   server-id: 5510-5513

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true

 

route:

   - source-table: positioning.device_beacon

     sink-table: bigdata_bi.ods_positioning_device_beacon

   - source-table: positioning.device_label

     sink-table: bigdata_bi.ods_positioning_device_label

   - source-table: positioning.device_station

     sink-table: bigdata_bi.ods_positioning_device_station

pipeline:

   name: route_positioning(定位设备信息表:beacon/label/station)

   parallelism: 2
 

 
mysql-to-starrocks_route_yx_energy.yaml

source:

   type: mysql

   hostname: 172.16.5.13

   port: 3305

   username: bigdata_repl

   password: bigdata_repl

   tables: yx_energy.device_info

   server-id: 5509

   server-time-zone: Asia/Shanghai

   #scan.startup.mode: initial

sink:

   type: starrocks

   jdbc-url: jdbc:mysql://172.21.235.202:9030

   load-url: 172.21.235.202:8030

   username: root

   password: "123456"

   table.create.properties.replication_num: 1

   table.create.properties.fast_schema_evolution: true
 

route:

   - source-table: yx_energy.device_info

     sink-table: bigdata_bi.ods_yx_energy_device_info

 
pipeline:

   name: route_yx_energy(能效设备信息表:device_info)

   parallelism: 2
 

 
二、启动实时同步任务

cd /usr/local/apps/flink-cdc-3.1.0/mysql-to-starrocks
export FLINK_HOME=/usr/local/apps/flink-1.18.0

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_evoc9_message_ms_alarmlevel.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_db_evoc_deviceinfo_alarm_info.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_evoc_platform.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_db_evoc9_person_ps_person.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_evoc9_person_ps_personvisit.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_db_evoc_deviceinfo_device_info.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_db_evoc_deviceinfo_device_type.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_devicex_app_device_info.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_positioning.yaml

bash bin/flink-cdc.sh ./mysql-to-starrocks/mysql-to-starrocks_route_yx_energy.yaml