Last active
December 22, 2018 05:38
-
-
Save ilyaevseev/a9bebe1ac19c91d597379db7de45ea78 to your computer and use it in GitHub Desktop.
Initialize Elasticsearch maps (database schema) for use it as Zabbix history storage.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh -e | |
| MAX_DAYS="93" | |
| LOG_DAYS="93" | |
| LOG_DIR="/var/log/zbx-es-metrics-cleanup" | |
| LOGFILE="$LOG_DIR/$(date +%Y-%m-%d-%H%M).log" | |
| mkdir -p "$LOG_DIR" | |
| D0="$(date -d "$MAX_DAYS days ago" +%s)" | |
| curl -sS "http://127.0.0.1:9200/_cat/indices" | awk '{ print $3 }' | | |
| while read idx; do | |
| D="$(date -d "${idx#*-}" +%s 2>/dev/null || :)" | |
| test "$D" = "" && continue | |
| test "$D" -ge "$D0" && continue | |
| curl -X DELETE "http://127.0.0.1:9200/$idx" | |
| done >> "$LOGFILE" 2>&1 | |
| for n in uint text str log dbl; do | |
| curl -sS -X POST "127.0.0.1:9200/$n/_delete_by_query?pretty=true" -H 'Content-Type: application/json' \ | |
| -d'{ "query": { "range" : { "clock" : { "lt" : "now-'"$MAX_DAYS"'d/d" } } } }' | |
| done >> "$LOGFILE" 2>&1 | |
| find "$LOG_DIR/" -type f -name '*.log' -mtime "+$LOG_DAYS" -delete | |
| ## END ## |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| Map_datebased_template() { | |
| local NAME="$1" VALUE="$2" | |
| curl -X PUT "http://127.0.0.1:9200/_template/${NAME}_template" \ | |
| -H 'content-type:application/json' -d '{ | |
| "template": "'"$NAME"'*", | |
| "index_patterns": ["'"$NAME"'*"], | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| '"$VALUE"' | |
| } | |
| } | |
| } | |
| } | |
| }' | |
| curl -X PUT "http://127.0.0.1:9200/_ingest/pipeline/$NAME-pipeline" \ | |
| -H 'content-type:application/json' -d '{ | |
| "description": "daily '"$NAME"' index naming", | |
| "processors": [ | |
| { | |
| "date_index_name": { | |
| "field": "clock", | |
| "date_formats": ["UNIX"], | |
| "index_name_prefix": "'"$NAME"'-", | |
| "date_rounding": "d" | |
| } | |
| } | |
| ] | |
| }' | |
| } | |
| Map_datebased_text_template() { | |
| Map_datebased_template "$1" ' | |
| "fields" : { | |
| "analyzed" : { | |
| "index" : true, | |
| "type" : "text", | |
| "analyzer" : "standard" | |
| } | |
| }, | |
| "index" : false, | |
| "type" : "text" | |
| ' | |
| } | |
| Map_datebased_template uint '"type" : "long"' | |
| Map_datebased_template dbl '"type" : "double"' | |
| Map_datebased_text_template str | |
| Map_datebased_text_template text | |
| Map_datebased_text_template log |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/sh | |
| # Based on https://github.com/zabbix/zabbix/blob/trunk/database/elasticsearch/elasticsearch.map | |
| # See also https://www.zabbix.com/documentation/4.0/manual/appendix/install/elastic_search_setup | |
| if test "$1" = "check"; then | |
| curl "127.0.0.1:9200/_all/_mapping?pretty=true" | |
| curl "127.0.0.1:9200/_cat/indices?v" | |
| exit | |
| fi | |
| Map() { | |
| local idx="$1"; shift | |
| curl -X PUT "http://127.0.0.1:9200/$idx" -H 'content-type: application/json' -d "$@" | |
| echo | |
| } | |
| Map uint ' | |
| { | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| "type" : "long" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ' | |
| Map dbl ' | |
| { | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| "type" : "double" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ' | |
| Map str ' | |
| { | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| "fields" : { | |
| "analyzed" : { | |
| "index" : true, | |
| "type" : "text", | |
| "analyzer" : "standard" | |
| } | |
| }, | |
| "index" : false, | |
| "type" : "text" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ' | |
| Map text ' | |
| { | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| "fields" : { | |
| "analyzed" : { | |
| "index" : true, | |
| "type" : "text", | |
| "analyzer" : "standard" | |
| } | |
| }, | |
| "index" : false, | |
| "type" : "text" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ' | |
| Map log ' | |
| { | |
| "settings" : { | |
| "index" : { | |
| "number_of_replicas" : 1, | |
| "number_of_shards" : 5 | |
| } | |
| }, | |
| "mappings" : { | |
| "values" : { | |
| "properties" : { | |
| "itemid" : { | |
| "type" : "long" | |
| }, | |
| "clock" : { | |
| "format" : "epoch_second", | |
| "type" : "date" | |
| }, | |
| "value" : { | |
| "fields" : { | |
| "analyzed" : { | |
| "index" : true, | |
| "type" : "text", | |
| "analyzer" : "standard" | |
| } | |
| }, | |
| "index" : false, | |
| "type" : "text" | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ' | |
| ## END ## |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment