Skip to content

C
cloud-esearch
Commit af9a3f22 by Administrator
Options

update

parent 9b53ec54
Showing with 1019 additions and 174 deletions
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-album.conf 0 → 100644
input {
stdin {
}
jdbc {
type => "album"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_album?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_album.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_album.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "album" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "album"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "album" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-caseLibrary.conf 0 → 100644
input {
stdin {
}
# caseLibrary start -----------------------------------------------------------------------------------------------------
jdbc {
type => "case_library"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_case_library?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_caseLibrary.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_caseLibrary.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
jdbc {
type => "case_library_auth"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_case_library?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_caseLibrary_relationIds.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_caseLibrary_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# caseLibrary end -----------------------------------------------------------------------------------------------------
# studentCase start -----------------------------------------------------------------------------------------------------
jdbc {
type => "student_case"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_case_library?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_studentCase.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_studentCase.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
jdbc {
type => "student_case_auth"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_case_library?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_studentCase_relationIds.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_studentCase_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# studentCase end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "student_case" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "student_case"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "student_case_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "student_case_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "student_case" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "case_library" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "case_library"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "case_library_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "case_library_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "case_library" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-course.conf 0 → 100644
input {
stdin {
}
# course start -----------------------------------------------------------------------------------------------------
# 课程
jdbc {
type => "course"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_course?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_course.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_course.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# 课程
jdbc {
type => "course_auth"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_course?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_course_relationIds.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_course_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# course end -----------------------------------------------------------------------------------------------------
# offlineCourse start -----------------------------------------------------------------------------------------------------
jdbc {
type => "offline_course"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_course?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_offlineCourse.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_offlineCourse.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# offlineCourse end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "course" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "course"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "course_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "course_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "course" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "offline_course" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "offline_course"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-exam.conf 0 → 100644
input {
stdin {
}
# exam start -----------------------------------------------------------------------------------------------------
jdbc {
type => "exam"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_exam?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_exam.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_exam.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
jdbc {
type => "exam_auth"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_exam?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_exam_relationIds.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_exam_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# exam end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "exam" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "exam"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "exam_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "exam_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "exam" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-lecturer.conf 0 → 100644
input {
stdin {
}
# lecturer start -----------------------------------------------------------------------------------------------------
# 物理删除，暂时无法做到近实时同步
jdbc {
type => "lecturer"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_lecturer?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_lecturer.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_lecturer.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# lecturer end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "lecturer" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "lecturer"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-live.conf 0 → 100644
input {
stdin {
}
# live start -----------------------------------------------------------------------------------------------------
jdbc {
type => "live"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_live?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_live.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_live.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# live end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "live" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "live"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "live_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "live_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "live" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-research.conf 0 → 100644
input {
stdin {
}
# research start -----------------------------------------------------------------------------------------------------
jdbc {
type => "research"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_research?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_research.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_research.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
jdbc {
type => "research_auth"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_research?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_research_relationIds.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_research_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
# research end -----------------------------------------------------------------------------------------------------
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "research" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "research"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "research_auth" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "research_auth"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "research" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-system.conf 0 → 100644
input {
stdin {
}
jdbc {
type => "account"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_system?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_account.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_account.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
jdbc {
type => "organization"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_system?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_organization.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_organization.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "* * * * *"
}
}
filter {
ruby {
code => "event.timestamp.time.localtime"
}
}
output {
if [type] == "organization" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "organization"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "account" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "account"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout {
codec => json_lines
}
}
cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/mysql-dev.conf → cloud-esearch-service/src/main/resources/logstash-config/mysql/mysql-dev/db-trainingProject.conf
...@@ -2,331 +2,286 @@ input { ...@@ -2,331 +2,286 @@ input {
stdin { stdin {
} }
# 课程 # assignment start -----------------------------------------------------------------------------------------------------
jdbc { jdbc {
type => "course" type => "assignment"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_course?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_course.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_assignment.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_course.txt" last_run_metadata_path => "./station_assignment.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# 培训项目
jdbc { jdbc {
type => "training_project" type => "assignment_auth"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_training_project.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_assignment_relationIds.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_training_project.txt" last_run_metadata_path => "./station_assignment_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# assignment end -----------------------------------------------------------------------------------------------------
# 作业 # training_project start -----------------------------------------------------------------------------------------------------
jdbc { jdbc {
type => "assignment" type => "training_project"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_assignment.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_training_project.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_assignment.txt" last_run_metadata_path => "./station_training_project.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# 考试
jdbc { jdbc {
type => "exam" type => "training_project_auth"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_exam?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_exam.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_training_project_relationIds.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_exam.txt" last_run_metadata_path => "./station_training_project_relationIds.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# 调研
jdbc { jdbc {
type => "research" type => "training_project_enroll_record"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_research?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_research.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/quety_training_project_record_enroll.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_research.txt" last_run_metadata_path => "./station_training_project_record_enroll.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# training_project end -----------------------------------------------------------------------------------------------------
# 组织部门 # biz_visible_range start -----------------------------------------------------------------------------------------------------
jdbc { jdbc {
type => "organization" type => "biz_visible_range"
# 数据库 # 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_system?tinyInt1isBit=false" jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_trainning_project?tinyInt1isBit=false"
jdbc_user => "fulandba" jdbc_user => "fulandba"
jdbc_password => "Fldba#13!" jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" jdbc_driver_library => "/opt/logstash-7.6.2/logstash-config/mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"} codec => plain { charset => "UTF-8"}
lowercase_column_names => false lowercase_column_names => false
statement_filepath => "./query_organization.sql" statement_filepath => "/opt/logstash-7.6.2/logstash-config/sql/query_training_project_visible_range.sql"
jdbc_paging_enabled => "true" jdbc_paging_enabled => "true"
jdbc_page_size => "2000" jdbc_page_size => "5000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value # 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false use_column_value => false
tracking_column => index_time tracking_column => index_time
record_last_run => true record_last_run => true
last_run_metadata_path => "./station_organization.txt" last_run_metadata_path => "./station_training_project_visible_range.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 # 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *" schedule => "* * * * *"
} }
# biz_visible_range end -----------------------------------------------------------------------------------------------------
# 用户 }
jdbc {
type => "account"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_system?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "./query_account.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "2000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_account.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 filter {
schedule => "*/5 * * * *" ruby {
code => "event.timestamp.time.localtime"
} }
}
# 讲师 output {
jdbc {
type => "lecturer"
# 数据库
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_lecturer?tinyInt1isBit=false"
jdbc_user => "fulandba"
jdbc_password => "Fldba#13!"
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "./query_lecturer.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "2000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_lecturer.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新 if [type] == "training_project" {
schedule => "*/5 * * * *" elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "training_project"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
} }
# 线下课程 if [type] == "training_project_auth" {
jdbc { elasticsearch {
type => "offline_course" hosts => "elasticsearch7:9200"
# index名
# 数据库 index => "training_project_auth"
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_course?tinyInt1isBit=false" # 需要关联的数据库中有有一个id字段，对应索引的id号
jdbc_user => "fulandba" document_id => "%{id}"
jdbc_password => "Fldba#13!" template_overwrite => true
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
jdbc_driver_class => "com.mysql.jdbc.Driver" }
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "./query_offlineCourse.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "2000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => index_time
record_last_run => true
last_run_metadata_path => "./station_offlineCourse.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *"
} }
# 案例库 if [type] == "training_project" {
jdbc { elasticsearch {
type => "student_case" hosts => "elasticsearch7:9200"
# index名
# 数据库 index => "doc_recommend"
jdbc_connection_string => "jdbc:mysql://192.168.10.32:3306/cloud_case_library?tinyInt1isBit=false" # 需要关联的数据库中有有一个id字段，对应索引的id号
jdbc_user => "fulandba" document_id => "%{recommendId}"
jdbc_password => "Fldba#13!" template_overwrite => true
jdbc_driver_library => "./mysql-connector-java-5.1.25.jar" template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
jdbc_driver_class => "com.mysql.jdbc.Driver" }
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
statement_filepath => "./query_studentCase.sql"
jdbc_paging_enabled => "true"
jdbc_page_size => "2000"
# 不使用字段值追踪，直接用该次更新的当前时间赋值到:sql_last_value，然后在sql中用创建时间和更新时间 >= :sql_last_value
use_column_value => false
tracking_column => update_time
record_last_run => true
last_run_metadata_path => "./station_studentCase.txt"
# 设置监听间隔 各字段含义（由左至右）分、时、天、月、年，全部为*默认含义为每分钟都更新
schedule => "*/5 * * * *"
} }
}
#filter { if [type] == "training_project_enroll_record" {
# json { elasticsearch {
# source => "message" hosts => "elasticsearch7:9200"
# remove_field => ["message"] # index名
# } index => "training_project_enroll_record"
#} # 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
output { if [type] == "assignment" {
if [type] == "course" {
elasticsearch { elasticsearch {
hosts => "elasticsearch7:9200" hosts => "elasticsearch7:9200"
# index名 # index名
index => "course" index => "assignment"
document_type => "course"
# 需要关联的数据库中有有一个id字段，对应索引的id号 # 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}" document_id => "%{id}"
template_overwrite => true template_overwrite => true
template => "./logstash.json" template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}, }
}
if [type] == "assignment_auth" {
elasticsearch { elasticsearch {
hosts => "elasticsearch7:9200" hosts => "elasticsearch7:9200"
# index名 # index名
index => "course" index => "assignment_auth"
document_type => "course"
# 需要关联的数据库中有有一个id字段，对应索引的id号 # 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}" document_id => "%{id}"
template_overwrite => true template_overwrite => true
template => "./logstash.json" template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
} }
}
if [type] == "assignment" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "doc_recommend"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{recommendId}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
if [type] == "biz_visible_range" {
elasticsearch {
hosts => "elasticsearch7:9200"
# index名
index => "biz_visible_range"
# 需要关联的数据库中有有一个id字段，对应索引的id号
document_id => "%{id}"
template_overwrite => true
template => "/opt/logstash-7.6.2/logstash-config/template/logstash.json"
}
}
stdout { stdout {
codec => json_lines codec => json_lines
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment