{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"2"},"outline":{"value":"false"},"type":{"value":"list"},"printable":{"value":"false"}},"macroMetadata":{"macroId":{"value":"86af0120-ce6b-4935-abf5-ccbf02a89ff0"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"13df9852-6966-49c3-8c1e-79685bff7f72"}},{"type":"paragraph","content":[{"text":"This is the process for in-place Elasticsearch (ES) upgrades, using an existing Search Feed and index data.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Required","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"This upgrade is for Elasticsearch 6.8.6 and 7.5.2 or higher with a Search Feed created on Swarm 11 or higher. ","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"If you are using Elasticsearch 6.8.6, see ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://perifery.atlassian.net/wiki/spaces/public/pages/2443804878/How+to+Upgrade+Swarm#Upgrading-from-Unsupported-Elasticsearch"}},{"text":" for a specific version and live-upgrade ordering requirements.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"See ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://perifery.atlassian.net/wiki/spaces/public/pages/2443809821"}},{"text":" for migrating from Elasticsearch 2.3.3 or 5.6.12.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Elasticsearch cannot be downgraded once it’s upgraded.","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Upgrading Elasticsearch by Script","type":"text"}]},{"type":"paragraph","content":[{"text":"On ","type":"text"},{"text":"each node","type":"text","marks":[{"type":"em"}]},{"text":" in an Elasticsearch cluster, follow this process and run the files from the Swarm download bundle:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Before upgrading, query the Elasticsearch cluster for the list of nodes.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"curl -i http://ELASTICSEARCH:9200/_cat/nodes","type":"text"}]},{"type":"paragraph","content":[{"text":"Example output:","type":"text"}]},{"type":"codeBlock","content":[{"text":"[root@elasticsearch Elasticsearch]# curl -i http://:9200/_cat/nodes\nHTTP/1.1 200 OK\ncontent-type: text/plain; charset=UTF-8\ncontent-length: 300\n 14 99 1 0.02 0.14 0.15 dilm - \n 1 71 1 0.01 0.09 0.08 dilm - \n 72 99 0 0.08 0.06 0.08 dilm - \n 50 99 0 0.03 0.04 0.05 dilm * \n 47 99 1 0.08 0.14 0.22 dilm - ","type":"text"}]},{"type":"paragraph","content":[{"text":"The node with * is the Elasticsearch master node. It is recommended to upgrade the node ","type":"text"},{"text":"last","type":"text","marks":[{"type":"em"}]},{"text":" to avoid problems electing a new master node.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Backup the existing configurations so a record exists of any customizations. Run ","type":"text"},{"text":"./techsupport-bundle-grab.sh","type":"text","marks":[{"type":"code"}]},{"text":" from your Support tools directory to backup to upgrade your support tools before running the script. Instructions are ","type":"text"},{"text":"here","type":"text","marks":[{"type":"link","attrs":{"href":"https://perifery.atlassian.net/wiki/spaces/public/pages/2443826431"}}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Install the latest Swarm Search i.e. ","type":"text"},{"text":"caringo-elasticsearch-search","type":"text","marks":[{"type":"code"}]},{"text":" RPM.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"yum install caringo-elasticsearch-search-VERSION.noarch.rpm","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Tip","type":"text"}]},{"type":"paragraph","content":[{"text":"The error: \"","type":"text"},{"text":"ES_PATH_CONF must be set to the configuration path chown: cannot access '/etc/elasticsearch/elasticsearch.keystore': No such file or directory","type":"text","marks":[{"type":"code"}]},{"text":"\" is displayed if Elasticsearch 7 RPM was inadvertently installed. Install the ","type":"text"},{"text":"caringo-elasticsearch-search","type":"text","marks":[{"type":"code"}]},{"text":" RPM to proceed.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Info","type":"text"}]},{"type":"paragraph","content":[{"text":"Run ","type":"text"},{"text":"ls -la /etc/elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" command to verify the directory and files (especially ","type":"text"},{"text":"elasticsearch.keystore","type":"text","marks":[{"type":"code"}]},{"text":") are owned by ","type":"text"},{"text":"root:elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" and the directory has setgid. Run ","type":"text"},{"text":"chown -R root:elasticsearch /etc/elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"chmod g+s /etc/elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" to correct the permissions.","type":"text"}]}]},{"type":"orderedList","attrs":{"order":4},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the script to install and configure the upgrade.","type":"text"},{"type":"hardBreak"},{"text":"If the script detects that Elasticsearch is installed and configured, it runs with ","type":"text"},{"text":"--upgrade","type":"text","marks":[{"type":"code"}]},{"text":" instead of configuring a new cluster, automating the Elasticsearch rolling upgrade steps (","type":"text"},{"type":"inlineCard","attrs":{"url":"https://www.elastic.co/guide/en/elasticsearch/reference/7.17/rolling-upgrades.html"}},{"text":").","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"/usr/share/caringo-elasticsearch-search/bin/configure_elasticsearch_with_swarm_search.py --esversion 8.15.1","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Important","type":"text"}]},{"type":"paragraph","content":[{"text":"The upgrade requires Internet access to download the Elasticsearch rpm. Place the ","type":"text"},{"text":"elasticsearch-*.rpm","type":"text","marks":[{"type":"code"}]},{"text":" in the current directory if Internet access is unavailable.","type":"text"}]}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Warning","type":"text"}]},{"type":"paragraph","content":[{"text":"The configure script requires ","type":"text"},{"text":"node.name","type":"text","marks":[{"type":"code"}]},{"text":" to be set in ","type":"text"},{"text":"/etc/elasticsearch/elasticsearch.yml","type":"text","marks":[{"type":"code"}]},{"text":". This is not the case with a Swarm Cluster Installer (SCI) deployed VM; uncomment the setting and set it to the default value of ","type":"text"},{"text":"${HOSTNAME}","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]}]},{"type":"orderedList","attrs":{"order":5},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Compare the backup file to the newly created ","type":"text"},{"text":"elasticsearch.yml","type":"text","marks":[{"type":"code"}]},{"text":", add any customizations needed, and restart ","type":"text"},{"text":"systemctl restart elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" if the configuration was modified.","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"note"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Note","type":"text"}]},{"type":"paragraph","content":[{"text":"thread_pool.write.queue_size: 1000","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"prometheus.indices","type":"text","marks":[{"type":"code"}]},{"text":" are no longer required.","type":"text"}]}]},{"type":"orderedList","attrs":{"order":6},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Verify that all nodes are accounted for, all shards are assigned, and the status is green.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"curl -i 'http://ELASTICSEARCH:9200/_cat/health?v'","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"The script updates the configuration files and restarts the service if the upgraded Elasticsearch is already installed.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Important","type":"text"}]},{"type":"paragraph","content":[{"text":"Type ","type":"text"},{"text":"Ctrl-C","type":"text","marks":[{"type":"strong"}]},{"text":" once when the upgrade script is stuck in retrying status checks and proceed to the next node after the script finishes if the cluster loses master during the upgrade process and does not recover. Review ","type":"text"},{"text":"/etc/elasticsearch/elasticsearch.yml","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"tail -F /var/log/elasticsearch/.log","type":"text","marks":[{"type":"code"}]},{"text":" for configuration errors.","type":"text"}]},{"type":"paragraph","content":[{"text":"The nodes elect the master and recover once all nodes have started on the upgraded Elasticsearch. Health status goes yellow, then eventually green. Re-enable shard allocation, otherwise ","type":"text"},{"text":"/_cat/health?v","type":"text","marks":[{"type":"code"}]},{"text":" stops at 50% with health status yellow.","type":"text"}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Upgrading Elasticsearch Manually","type":"text"}]},{"type":"paragraph","content":[{"text":"The script automates the following steps for upgrading Elasticsearch manually:","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It fixes ","type":"text"},{"text":"/etc/sysconfig/elasticsearch","type":"text","marks":[{"type":"code"}]},{"text":" to the correct ES6 version (the same as ES7+).","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It increases the ","type":"text"},{"text":"systemd","type":"text","marks":[{"type":"code"}]},{"text":" timeout in ","type":"text"},{"text":"/etc/systemd/system/elasticsearch.service.d/override.conf","type":"text","marks":[{"type":"code"}]},{"text":" (see ","type":"text"},{"text":"github.com/elastic/elasticsearch/issues/60140","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/elastic/elasticsearch/issues/60140"}}]},{"text":")","type":"text"},{"type":"hardBreak"},{"text":"and does a ","type":"text"},{"text":"systemctl daemon-reload","type":"text","marks":[{"type":"code"}]},{"text":" to pick up configuration file changes.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"A prompt to continue with the yum upgrade to the latest ES version appears after refreshing the config files for Elasticsearch 6.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It disables shard allocation and performs a POST synced-flush for safer rolling upgrades. ","type":"text"}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"heading","attrs":{"level":3},"content":[{"text":"Important","type":"text"}]},{"type":"paragraph","content":[{"text":"Disabling shard allocation or sync-flush can fail to contact the node but does not proceed to upgrade the next node until the cluster health is green again.","type":"text"}]}]},{"type":"orderedList","attrs":{"order":5},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It uninstalls the existing Prometheus Exporter plugin.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It shells out to yum to install the Elasticsearch 7 RPM in the current directory or from ","type":"text"},{"text":"artifacts.elastic.co","type":"text","marks":[{"type":"link","attrs":{"href":"https://artifacts.elastic.co"}}]},{"text":", if unavailable.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It updates elasticsearch.yml for Elasticsearch 7+ compatibility, including ","type":"text"},{"text":"cluster.initial_master_nodes","type":"text","marks":[{"type":"code"}]},{"text":" instead of ","type":"text"},{"text":"discovery.zen.unicast.hosts","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"jvm.options","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"It starts the upgraded Elasticsearch and waits for it to be ready.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"The cluster re-enables shard allocation and prompts to repeat these two steps on the next node if the cluster health is green or yellow.","type":"text"}]}]}]}],"version":1}

Browser not supported