go build -o ./bin/main main.go计划任务:
go build -o ./bin/news-check ./app/console/news_check.gomysql: 阿里云RDS
node.name: node-1filebeat: /home/elk/elastic/filebeat-7.2.0/filebeat.yml
network.host: 0.0.0.0
http.port: 9200
xpack.security.enabled: true
xpack.security.authc.accept_default_password: false
discovery.type: single-node
#================================ Outputs =====================================kibana: /home/elk/elastic/kibana-7.2.0/config/kibana.yml
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
# Array of hosts to connect to.
hosts: ["0.0.0.0"]
# Optional protocol and basic auth credentials.
#protocol: "https"
username: "xxx"
password: "xxx"
server.port: 5601canal:mysql数据同步到es的工具
server.host: "0.0.0.0"
elasticsearch.hosts: ******: "xxx"
elasticsearch.password: "xxx"
xpack.security.encryptionKey: "something_at_least_32_characters"
# position infocanal客户端配置:
canal.instance.master.address=xxx:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=
# username/password
canal.instance.dbUsername=xxx
canal.instance.dbPassword=xxx
canal.instance.connectionCharset = UTF-8
# table regex
canal.instance.filter.regex=xxx.xxx
srcDataSources:/web/canal/1.1.5/adapter/conf/es7/article.yml
defaultDS:
url: jdbc:mysql://xxx:3306/xxx?useUnicode=true
username: canal
password: xxx
canalAdapters:
- instance: example # canal instance Name or mq topic name
groups:
- groupId: g1
outerAdapters:
- name: logger
- name: es7
hosts: xxx:9200 # 127.0.0.1:9200 for rest mode
properties:
mode: rest # transport # or rest
security.auth: username:password # only used for rest mode
cluster.name: elasticsearch # my-application # elasticsearch
dataSourceKey: defaultDS # 源数据源的key, 对应上面配置的srcDataSources中的值后续会持续更新。
destination: example # canal的instance或者MQ的topic
groupId: g1 # 对应MQ模式下的groupId, 只会同步对应groupId的数据
esMapping:
_index: article_xxx # es 的索引名称
_id: _id # es 的_id, 如果不配置该项必须配置下面的pk项_id则会由es自动分配
sql: "SELECT
a.id AS _id,
a.newsid,
a.uuid,
a.userid,
a.parentid,
a.catid,
a.title,
a.author,
a.keyword,
a.thumbid,
a.status,
a.inputtime,
a.updatetime,
a.ctime
FROM
article AS a"
#etlCondition: "where a.updatetime>={}" #etl的条件参数
commitBatch: 1000 # 提交批大小
* 文章为作者独立观点,不代表爱提网立场
* 本文不构成投资建议
转载此文请于文首标明作者姓名,保持文章完整性,并请附上出处(爱提网)及本页链接。原文链接:
https://www.itgoodboy.com/p/d6d9bdeba479
产品建议及投诉请联系:service@itgoodboy.com