Elasticsearch安装
大约 7 分钟
网址
- 官网:http://elasticsearch.co/
- 中文社区:https://elasticsearch.cn/
- 官网下载地址:https://www.elastic.co/cn/downloads/elasticsearch
- 中文社区下载地址:https://elasticsearch.cn/download/
- 华为云下载地址:https://repo.huaweicloud.com/elasticsearch/
- 百度网盘下载地址:https://pan.baidu.com/s/1Tdsgm9608fyt_j7gMsLZhg 提取码:e2nd
Elasticsearch安装
- 安装命令
# 新建用户
useradd dev -s /bin/bash
# 解压并放到dev用户目录下
tar -xzvf elasticsearch-7.10.2-linux-x86_64.tar.gz -C /home/dev/
# 修改解压文件的文件权限
chown -R dev:dev /home/dev/elasticsearch-7.10.2
# 查看文件权限
ll /home/dev/elasticsearch-7.10.2/
# 切换用户
su dev
# 启动ES,-d:后台启动
cd /home/dev/elasticsearch-7.10.2/bin && ./elasticsearch -d
# 测试启动成功
curl http://127.0.0.1:9200
- 修改配置
# 默认只能127.0.0.1访问,开放访问方式
sed -i "s/#\?network.host:.*/network.host: 0.0.0.0/g" /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
# 设置节点名称
sed -i "s/#\?node.name:.*/node.name: node-101/g" /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
# 集群配置
sed -i 's/#\?cluster.initial_master_nodes:.*/cluster.initial_master_nodes: ["node-101"]/g' /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
# 跨域处理,处理插件head的连接
cat << EOF >> /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
http.cors.enabled: true
http.cors.allow-origin: "*"
EOF
# 启动ES,-d:后台启动
cd /home/dev/elasticsearch-7.10.2/bin && ./elasticsearch -d
# 开启防火墙
firewall-cmd --zone=public --add-port=9200/tcp --permanent && firewall-cmd --reload
ES安装问题集
java.lang.RuntimeException: can not run elasticsearch as root
ES不能在root用户下启动,必须创建新的用户,用来启动ES
Exception in thread "main" java.nio.file.AccessDeniedException: /home/dev/elasticsearch-7.10.2/config/jvm.options.d
# 文件权限问题
chown -R dev:dev /home/dev/elasticsearch-7.10.2
max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]
# 每个进程最大同时打开文件数太小,可通过下面2个命令查看当前数量
# 查看硬限制
ulimit -Hn
# 查看软限制
ulimit -Sn
# 查看两个中更小的限制(软限制始终比硬限制低, 所以查看的是软限制)
ulimit -n
# 修改配置
cat << EOF >> /etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
EOF
max number of threads [3720] for user [dev] is too low, increase to at least [4096]
ulimit -Hu
ulimit -Su
# 修改配置
cat << EOF >> /etc/security/limits.conf
* soft nproc 4096
* hard nproc 4096
EOF
max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
cat << EOF >> /etc/sysctl.conf
vm.max_map_count=262144
EOF
# 使/etc/sysctl.conf的修改生效
sysctl -p
the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured
sed -i "s/#\?node.name:.*/node.name: node-101/g" /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
sed -i 's/#\?cluster.initial_master_nodes:.*/cluster.initial_master_nodes: ["node-101"]/g' /home/dev/elasticsearch-7.10.2/config/elasticsearch.yml
docker容器安装
#搜索镜像
docker search elasticsearch
#拉取镜像
docker pull elasticsearch:7.10.2
#创建容器
docker create name elasticsearch net host -e "discovery.type=singlenode" -e "network.host=192.168.66.66" elasticsearch:7.10.2
#启动
docker start elasticsearch
#查看日志
docker logs elasticsearch
Elasticsearch开机自启
方法一:systemctl设置开机自启(推荐)
cat << EOF >> /etc/systemd/system/elasticsearch.service
[Unit]
Description=elasticsearch
[Service]
User=dev
LimitNOFILE=100000
LimitNPROC=100000
ExecStart=/home/dev/elasticsearch-7.10.2/bin/elasticsearch
[Install]
WantedBy=multi-user.target
EOF
# 开机自启
systemctl enable elasticsearch.service
# 开启防火墙
firewall-cmd --zone=public --add-port=9200/tcp --permanent && firewall-cmd --reload
方法二:chkconfig设置开机自启
# 查看当前的开机启动服务
chkconfig --list
# ES启动脚本
cat << EOF >> /etc/init.d/elasticsearch
#!/bin/bash
#chkconfig: 345 63 37
#description: elasticsearch
#processname: elasticsearch-7.10.2
export ES_HOME=/home/dev/elasticsearch-7.10.2 #【这个目录是你Es所在文件夹的目录】
export ES_USER=dev #【es 这个是启动es的账户,如果你的不是这个记得调整】
case $1 in
start)
su - $ES_USER -c "$ES_HOME/bin/elasticsearch -d -p pid"
echo "elasticsearch is started"
;;
stop)
pid=`cat $ES_HOME/pid`
kill -9 $pid
echo "elasticsearch is stopped"
;;
restart)
pid=`cat $ES_HOME/pid`
kill -9 $pid
echo "elasticsearch is stopped"
sleep 1
su - $ES_USER -c "$ES_HOME/bin/elasticsearch -d -p pid"
echo "elasticsearch is started"
;;
*)
echo "start|stop|restart"
;;
esac
exit 0
EOF
# 修改文件权限
chmod 755 /etc/init.d/elasticsearch
# {添加|删除}系统服务
chkconfig --{add|del} elasticsearch
# {启动|停止|重启}ES服务
service elasticsearch {start|stop|restart}
# {开启关闭}开机自启
chkconfig elasticsearch {on|off}
# 查看ES进程
ps -ef | grep elasticsearch
head插件安装
- 用途
- 安装
#下载nodejs,head插件运行依赖node
wget https: nodejs.org/dist/v9.9.0/node-v9.9.0-linux-x64.tar.xz
# 解压
tar -xf node-v9.9.0-linux-x64.tar.xz
# 重命名
mv node-v9.9.0-linux-x64 /usr/local/node
# 配置文件
vim /etc/profile
# 将node的路径添加到path中
export PATH=$PATH:$JAVA_HOME/bin:/usr/local/node/bin
# 刷新配置
source /etc/profile
# 查询node版本,同时查看是否安装成功
node -v
# 下载head插件
wget https://github.com/mobz/elasticsearch-head/archive/master.zip
# 解压
unzip master.zip
# 使用淘宝的镜像库进行下载,速度很快
npm install -g cnpm registry=https://registry.npm.taobao.org
# 进入head插件解压目录,执行安装命令
cnpm install
# 启动head插件
npm start
# 启动head插件,或者使用
grunt server
# 如果是linux上允许按需开放端口
firewall-cmd --zone=public --add-port=9100/tcp --permanent && firewall-cmd --reload
- linux设置开机自启
# 启动脚本
cat << EOF >> /root/elasticsearch-head/start.sh
#!/bin/bash
export ES_HOME=/root/elasticsearch-head
export NODE_HOME=/usr/local/lib/nodejs
export PATH=$PATH:$NODE_HOME/bin
cd \$ES_HOME
nohup npm start > /dev/null 2>&1 &
cd -
EOF
chmod +x /root/elasticsearch-head/start.sh
# 添加脚本文件
echo "/bin/bash /root/elasticsearch-head/start.sh" >> /etc/rc.local
# 设置可执行权限
chmod +x /etc/rc.local
LogStash安装
# 解压
tar -zxvf logstash-7.10.2.tar.gz
# 启动 基本的 intput output ,stdin stdout 输入输出插件
bin/logstash -e 'input{ stdin{} } output{ stdout{} }'
# codec
bin/logstash -e 'input{ stdin{} } output{ stdout{ codec json } }'
# 日志内容写入elasticsearch
bin/logstash -e 'input{ stdin{} } output{ elasticsearch{ hosts => ["127.0.0.1:9200"] } }'
# 日志内容写入elasticsearch,同时输出
bin/logstash -e 'input{ stdin{} } output{ elasticsearch{ hosts => ["127.0.0.1:9200"] } stdout { ["127.0.0.1:9200"] } stdout{} }'
1. 配置语法,日志内容写入elasticsearch,同时输出
cat << EOF >> config/file.conf
input { stdin { } }
output {
elasticsearch { hosts => ["127.0.0.1:9200"] }
stdout { codec => rubydebug }
}
EOF
#启动命令
bin/logstash -f config/file.conf
2. file日志收集
cat << EOF >> config/file.conf
input {
file{
path => "/var/log/messages" #收集messages文件日志
type => "system"
start_position => "beginning" #记录上次收集的位置
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"] #写入elasticsearch的地址
index => "system-%{+YYYY.MM.dd}" #定义索引的名称
}
stdout { codec => rubydebug }
}
EOF
bin/logstash -f config/file.conf
3. Java日志收集
cat << EOF >> config/file.conf
input {
file{
path => "/var/log/messages"
type => "system"
start_position => "beginning"
}
# 加一个file文件收集日志插件,收集elasticsearch日志、es就是java语言开发的。
file{
path => "/home/es/elasticsearch-7.10.2/logs/elasticsearch.log"
type => "es-info"
start_position => "beginning"
# 目前导入日志都是按照行导入的、但是有些日志多行是一句话,如果分开的话,就不太容查看日志完整的意思了。
# 使用正则表达式,合并多行日志
codec => multiline {
# 发现中括号,就合并日志
pattern => "^\["
negate => true
what => "previous"
}
}
}
output {
if [type] "system"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "system-%{+YYYY.MM.dd}"
}
}
# 判断,导入到不同的索引库,否则会放入同一个索引库中
if [type] "es-info"{
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "es-info-%{+YYYY.MM.dd}"
}
}
stdout { codec => rubydebug }
}
EOF
bin/logstash -f config/file.conf
4. 项目日志
cat << EOF >> config/file.conf
# 通过tcp协议输入
input {
tcp {
port => 9600
codec => json
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "kkb-log-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}
EOF
bin/logstash -f config/file.conf
kibana安装
注意elasticsearch版本应与kibana版本一致,否则报错
kibana也是需要非root用户启动
tar -xzvf kibana-7.10.2-linux-x86_64.tar.gz -C /home/dev
chown -R dev:dev /home/dev/kibana-7.10.2-linux-x86_64/
# 切换到dev用户
su dev
# 修改配置
sed -i 's/#\?server.host:.*/server.host: "0.0.0.0"/g' /home/dev/kibana-7.10.2-linux-x86_64/config/kibana.yml
sed -i 's/#\?elasticsearch.hosts:.*/elasticsearch.hosts: ["http:\/\/192.16.18.101:9200"]/g' /home/dev/kibana-7.10.2-linux-x86_64/config/kibana.yml
# 启动
./kibana &
# 开启防火墙
firewall-cmd --zone=public --add-port=5601/tcp --permanent && firewall-cmd --reload
# 可浏览器查看
curl http://192.16.18.101:5601
cat << EOF >> /etc/systemd/system/kibana.service
[Unit]
Description=kibana
After=elasticsearch.service
[Service]
User=dev
LimitNOFILE=100000
LimitNPROC=100000
ExecStart=/home/dev/kibana-7.10.2-linux-x86_64/bin/kibana
[Install]
WantedBy=multi-user.target
EOF
# 开机自启
systemctl enable kibana.service
中文分词器
下载地址:https://github.com/medcl/elasticsearch-analysis-ik/releases
ik有两种分词模式:ik_max_word(细粒度的拆分)、ik_smart(粗粒度的拆分)。
wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.10.2/elasticsearch-analysis-ik-7.10.2.zip
mv elasticsearch-analysis-ik-7.10.2.zip /home/dev/
chown -R dev:dev /home/dev/elasticsearch-analysis-ik-7.10.2.zip
su dev
unzip elasticsearch-analysis-ik-7.10.2.zip -d elasticsearch-7.10.2/plugins/ik_analyze/
# 重启elasticsearch
systemctl status elasticsearch.service
# 测试分词结果
curl -XPOST 'localhost:9200/_analyze?pretty' -H 'Content-Type: application/json' -d '{"analyzer": "ik_max_word","text": "明星所在的娱乐圈真热闹"}'
拼音分词器
https://github.com/medcl/elasticsearch-analysis-pinyin/releases/tag/v6.6.2
# 测试分词结果
curl -XPOST 'localhost:9200/medcl/_analyze?pretty' -H 'Content-Type: application/json' -d '{"analyzer": "pinyin_analyzer","text": "刘德华"}'
集群部署
准备三台Elaticsearch服务器 ==》 修改配置如下 ==》 启动每个节点 ==》 查看集群情况
初始化集群部署时,保证各节点的数据为空
cat << EOF >> elasticsearch-7.10.2/config/elasticsearch.yml
# 集群名称,保证唯一
cluster.name: es-cluster-custom
# 节点名称,必须不一样
node.name: node-101
# 必须为本机的ip地址
network.host: 192.16.18.101
# 服务端口号,在同一机器下必须不一样
http.port: 9200
# 集群间通信端口号,在同一机器下必须不一样
transport.tcp.port: 9300
# 设置集群自动发现机器ip集合
discovery.seed_hosts: ["192.16.18.101:9300", "192.16.18.102:9300","192.16.18.103:9300"]
# 可当选master节点的节点名称集合
cluster.initial_master_nodes: ["node-101","node-102","node-103"]
# 跨域访问
http.cors.enabled: true
http.cors.allow-origin: "*"
EOF
# 每个节点开放节点通信端口(9300)
firewall-cmd --zone=public --add-port=9300/tcp --permanent && firewall-cmd --reload