#!/bin/bash
#
# Copyright (c) 2014 David Vossel <davidvossel@gmail.com>
#                    All Rights Reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it would be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# Further, this software is distributed without any warranty that it is
# free of the rightful claim of any third person regarding infringement
# or the like.  Any license provided herein, whether implied or
# otherwise, applies only to this software file.  Patent licenses, if
# any, provided herein do not apply to combinations of this program with
# other software, or any other product whatsoever.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write the Free Software Foundation,
# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
#

##
# README.
# 
# This agent only supports being configured as a multistate Master
# resource.
#
# Slave vs Master role:
#
# During the 'Slave' role, galera instances are in read-only mode and
# will not attempt to connect to the cluster. This role exists only as
# a means to determine which galera instance is the most up-to-date. The
# most up-to-date node will be used to bootstrap a galera cluster that
# has no current members.
#
# The galera instances will only begin to be promoted to the Master role
# once all the nodes in the 'wsrep_cluster_address' connection address
# have entered read-only mode. At that point the node containing the
# database that is most current will be promoted to Master. Once the first
# Master instance bootstraps the galera cluster, the other nodes will be
# promoted to Master as well.
#
# Example: Create a galera cluster using nodes rhel7-node1 rhel7-node2 rhel7-node3
#
# pcs resource create db galera enable_creation=true \
# wsrep_cluster_address="gcomm://rhel7-auto1,rhel7-auto2,rhel7-auto3" meta master-max=3 --master
#
# By setting the 'enable_creation' option, the database will be automatically 
# generated at startup. The meta attribute 'master-max=3' means that all 3
# nodes listed in the wsrep_cluster_address list will be allowed to connect
# to the galera cluster and perform replication.
#
# NOTE: If you have more nodes in the pacemaker cluster then you wish
# to have in the galera cluster, make sure to use location contraints to prevent
# pacemaker from attempting to place a galera instance on a node that is
# not in the 'wsrep_cluster_address" list. 
#
##

#######################################################################
# Initialization:

: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs

if [ "$__OCF_ACTION" != "meta-data" ]; then
    . ${OCF_FUNCTIONS_DIR}/mysql-common.sh
    NODENAME=$(ocf_attribute_target)
fi

# It is common for some galera instances to store
# check user that can be used to query status
# in this file
if [ -f "/etc/sysconfig/clustercheck" ]; then
    . /etc/sysconfig/clustercheck
elif [ -f "/etc/default/clustercheck" ]; then
    . /etc/default/clustercheck
fi

# Parameter defaults

OCF_RESKEY_wsrep_cluster_address_default=""
OCF_RESKEY_cluster_host_map_default=""
OCF_RESKEY_check_user_default=""
OCF_RESKEY_check_passwd_default=""
OCF_RESKEY_two_node_mode_default="false"

: ${OCF_RESKEY_wsrep_cluster_address=${OCF_RESKEY_wsrep_cluster_address_default}}
: ${OCF_RESKEY_cluster_host_map=${OCF_RESKEY_cluster_host_map_default}}
: ${OCF_RESKEY_check_user=${OCF_RESKEY_check_user_default}}
: ${OCF_RESKEY_check_passwd=${OCF_RESKEY_check_passwd_default}}
: ${OCF_RESKEY_two_node_mode=${OCF_RESKEY_two_node_mode_default}}

#######################################################################
# Defaults:

OCF_RESKEY_check_passwd_use_empty_default=0

: ${OCF_RESKEY_check_passwd_use_empty=${OCF_RESKEY_check_passwd_use_empty_default}}

#######################################################################

usage() {
  cat <<UEND
usage: $0 (start|stop|validate-all|meta-data|monitor|promote|demote)

$0 manages a galera Database as an HA resource.

The 'start' operation starts the database.
The 'stop' operation stops the database.
The 'status' operation reports whether the database is running
The 'monitor' operation reports whether the database seems to be working
The 'promote' operation makes this mysql server run as master
The 'demote' operation makes this mysql server run as slave
The 'validate-all' operation reports whether the parameters are valid

UEND
}

meta_data() {
   cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="galera" version="1.0">
<version>1.0</version>

<longdesc lang="en">
Resource script for managing galera database.
</longdesc>
<shortdesc lang="en">Manages a galera instance</shortdesc>
<parameters>

<parameter name="binary" unique="0" required="0">
<longdesc lang="en">
Location of the MySQL server binary
</longdesc>
<shortdesc lang="en">MySQL server binary</shortdesc>
<content type="string" default="${OCF_RESKEY_binary_default}" />
</parameter>

<parameter name="client_binary" unique="0" required="0">
<longdesc lang="en">
Location of the MySQL client binary
</longdesc>
<shortdesc lang="en">MySQL client binary</shortdesc>
<content type="string" default="${OCF_RESKEY_client_binary_default}" />
</parameter>

<parameter name="config" unique="0" required="0">
<longdesc lang="en">
Configuration file
</longdesc>
<shortdesc lang="en">MySQL config</shortdesc>
<content type="string" default="${OCF_RESKEY_config_default}" />
</parameter>

<parameter name="datadir" unique="0" required="0">
<longdesc lang="en">
Directory containing databases
</longdesc>
<shortdesc lang="en">MySQL datadir</shortdesc>
<content type="string" default="${OCF_RESKEY_datadir_default}" />
</parameter>

<parameter name="user" unique="0" required="0">
<longdesc lang="en">
User running MySQL daemon
</longdesc>
<shortdesc lang="en">MySQL user</shortdesc>
<content type="string" default="${OCF_RESKEY_user_default}" />
</parameter>

<parameter name="group" unique="0" required="0">
<longdesc lang="en">
Group running MySQL daemon (for logfile and directory permissions)
</longdesc>
<shortdesc lang="en">MySQL group</shortdesc>
<content type="string" default="${OCF_RESKEY_group_default}"/>
</parameter>

<parameter name="log" unique="0" required="0">
<longdesc lang="en">
The logfile to be used for mysqld.
</longdesc>
<shortdesc lang="en">MySQL log file</shortdesc>
<content type="string" default="${OCF_RESKEY_log_default}"/>
</parameter>

<parameter name="pid" unique="0" required="0">
<longdesc lang="en">
The pidfile to be used for mysqld.
</longdesc>
<shortdesc lang="en">MySQL pid file</shortdesc>
<content type="string" default="${OCF_RESKEY_pid_default}"/>
</parameter>

<parameter name="socket" unique="0" required="0">
<longdesc lang="en">
The socket to be used for mysqld.
</longdesc>
<shortdesc lang="en">MySQL socket</shortdesc>
<content type="string" default="${OCF_RESKEY_socket_default}"/>
</parameter>

<parameter name="enable_creation" unique="0" required="0">
<longdesc lang="en">
If the MySQL database does not exist, it will be created
</longdesc>
<shortdesc lang="en">Create the database if it does not exist</shortdesc>
<content type="boolean" default="${OCF_RESKEY_enable_creation_default}"/>
</parameter>

<parameter name="additional_parameters" unique="0" required="0">
<longdesc lang="en">
Additional parameters which are passed to the mysqld on startup.
(e.g. --skip-external-locking or --skip-grant-tables)
</longdesc>
<shortdesc lang="en">Additional parameters to pass to mysqld</shortdesc>
<content type="string" default="${OCF_RESKEY_additional_parameters_default}"/>
</parameter>


<parameter name="wsrep_cluster_address" unique="0" required="1">
<longdesc lang="en">
The galera cluster address. This takes the form of:
gcomm://node,node,node

Only nodes present in this node list will be allowed to start a galera instance.
The galera node names listed in this address are expected to match valid
pacemaker node names. If both names need to differ, you must provide a
mapping in option cluster_host_map.
</longdesc>
<shortdesc lang="en">Galera cluster address</shortdesc>
<content type="string" default="${OCF_RESKEY_wsrep_cluster_address_default}"/>
</parameter>

<parameter name="cluster_host_map" unique="0" required="0">
<longdesc lang="en">
A mapping of pacemaker node names to galera node names.

To be used when both pacemaker and galera names need to differ,
(e.g. when galera names map to IP from a specific network interface)
This takes the form of:
pcmk1:node.1.galera;pcmk2:node.2.galera;pcmk3:node.3.galera

where the galera resource started on node pcmk1 would be named
node.1.galera in the wsrep_cluster_address
</longdesc>
<shortdesc lang="en">Pacemaker to Galera name mapping</shortdesc>
<content type="string" default="${OCF_RESKEY_cluster_host_map_default}"/>
</parameter>

<parameter name="check_user" unique="0" required="0">
<longdesc lang="en">
Cluster check user.
</longdesc>
<shortdesc lang="en">MySQL test user</shortdesc>
<content type="string" default="${OCF_RESKEY_check_user_default}" />
</parameter>

<parameter name="check_passwd" unique="0" required="0">
<longdesc lang="en">
Cluster check user password. Empty passwords are ignored unless
the parameter "check_passwd_use_empty" is set to 1.
</longdesc>
<shortdesc lang="en">check password</shortdesc>
<content type="string" default="${OCF_RESKEY_check_passwd_default}" />
</parameter>

<parameter name="check_passwd_use_empty" unique="0" required="0">
<longdesc lang="en">
Use an empty "check_passwd" password. If this parameter is set to 1,
"check_passwd" will be ignored and an empty password is used
when calling the "mysql" client binary.
</longdesc>
<shortdesc lang="en">check password use empty</shortdesc>
<content type="boolean" default="${OCF_RESKEY_check_passwd_use_empty_default}"/>
</parameter>

<parameter name="two_node_mode" unique="0" required="0">
<longdesc lang="en">
If running in a 2-node pacemaker cluster, rely on pacemaker quorum
to allow automatic recovery even when the other node is unreachable.
Use it with caution! (and fencing)
</longdesc>
<shortdesc lang="en">Special recovery when running on a 2-node cluster</shortdesc>
<content type="boolean" default="${OCF_RESKEY_two_node_mode_default}"/>
</parameter>

</parameters>

<actions>
<action name="start" timeout="120s" />
<action name="stop" timeout="120s" />
<action name="status" timeout="60s" />
<action name="monitor" depth="0" timeout="30s" interval="20s" />
<action name="monitor" role="Master" depth="0" timeout="30s" interval="10s" />
<action name="monitor" role="Slave" depth="0" timeout="30s" interval="30s" />
<action name="promote" timeout="300s" />
<action name="demote" timeout="120s" />
<action name="validate-all" timeout="5s" />
<action name="meta-data" timeout="5s" />
</actions>
</resource-agent>
END
}

get_option_variable()
{
    local key=$1

    $MYSQL $MYSQL_OPTIONS_CHECK  -e "SHOW VARIABLES like '$key';" | tail -1
}

get_status_variable()
{
    local key=$1

    $MYSQL $MYSQL_OPTIONS_CHECK -e "show status like '$key';" | tail -1
}

set_bootstrap_node()
{
    local node=$(ocf_attribute_target $1)

    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -v "true"
}

clear_bootstrap_node()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" -D
}

is_bootstrap()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-bootstrap" --quiet 2>/dev/null

}

set_no_grastate()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -v "true"
}

clear_no_grastate()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" -D
}

is_no_grastate()
{
    local node=$(ocf_attribute_target $1)
    ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-no-grastate" --quiet 2>/dev/null
}

clear_last_commit()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -D
}

set_last_commit()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" -v $1
}

get_last_commit()
{
    local node=$(ocf_attribute_target $1)

    if [ -z "$node" ]; then
       ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
    else 
       ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-last-committed" --quiet 2>/dev/null
    fi
}

clear_safe_to_bootstrap()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -D
}

set_safe_to_bootstrap()
{
    ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" -v $1
}

get_safe_to_bootstrap()
{
    local node=$(ocf_attribute_target $1)

    if [ -z "$node" ]; then
        ${HA_SBIN_DIR}/crm_attribute -N $NODENAME -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
    else
        ${HA_SBIN_DIR}/crm_attribute -N $node -l reboot --name "${INSTANCE_ATTR_NAME}-safe-to-bootstrap" --quiet 2>/dev/null
    fi
}

wait_for_sync()
{
    local state=$(get_status_variable "wsrep_local_state")

    ocf_log info "Waiting for database to sync with the cluster. "
    while [ "$state" != "4" ]; do
        sleep 1
        state=$(get_status_variable "wsrep_local_state")
    done
    ocf_log info "Database synced."
}

is_primary()
{
    cluster_status=$(get_status_variable "wsrep_cluster_status")
    if [ "$cluster_status" = "Primary" ]; then
        return 0
    fi

    if [ -z "$cluster_status" ]; then
        ocf_exit_reason "Unable to retrieve wsrep_cluster_status, verify check_user '$OCF_RESKEY_check_user' has permissions to view status"
    else
        ocf_log info "Galera instance wsrep_cluster_status=${cluster_status}"
    fi
    return 1
}

is_readonly()
{
    local res=$(get_option_variable "read_only")

    if ! ocf_is_true "$res"; then
        return 1
    fi

    cluster_status=$(get_status_variable "wsrep_cluster_status")
    if ! [ "$cluster_status" = "Disconnected" ]; then
        return 1
    fi

    return 0
}

is_two_node_mode_active()
{
    # crm_node or corosync-quorumtool cannot access various corosync
    # flags when running inside a bundle, so only count the cluster
    # members
    ocf_is_true "$OCF_RESKEY_two_node_mode" && crm_mon_no_validation -1X | xmllint --xpath "count(//nodes/node[@type='member'])" - | grep -q -w 2
}

is_last_node_in_quorate_partition()
{
    # when a network split occurs in a 2-node cluster, pacemaker
    # fences the other node and try to retain quorum. So until
    # the fencing is resolved (and the status of the peer node
    # is clean), we shouldn't consider ourself quorate.
    local partition_members=$(${HA_SBIN_DIR}/crm_node -p | wc -w)
    local quorate=$(${HA_SBIN_DIR}/crm_node -q)
    local clean_members=$(crm_mon_no_validation -1X | xmllint --xpath 'count(//nodes/node[@type="member" and @unclean="false"])' -)

    [ "$partition_members" = 1 ] && [ "$quorate" = 1 ] && [ "$clean_members" = 2 ]
}

master_exists()
{
    if [ "$__OCF_ACTION" = "demote" ]; then
        # We don't want to detect master instances during demote.
        # 1. we could be detecting ourselves as being master, which is no longer the case.
        # 2. we could be detecting other master instances that are in the process of shutting down.
        # by not detecting other master instances in "demote" we are deferring this check
        # to the next recurring monitor operation which will be much more accurate
        return 1
    fi
    # determine if a master instance is already up and is healthy
    ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.1.0"
    res=$?
    if [ -z "$OCF_RESKEY_crm_feature_set" ] || [ $res -eq 2 ]; then
        XMLOPT="--output-as=xml"
        ocf_version_cmp "$OCF_RESKEY_crm_feature_set" "3.2.0"
        if [ $? -eq 1 ]; then
            crm_mon_no_validation -1 $XMLOPT >/dev/null 2>&1
            if [ $? -ne 0 ]; then
                XMLOPT="--as-xml"
            fi
        fi
    else
        XMLOPT="--as-xml"
    fi
    crm_mon_no_validation -1 $XMLOPT | grep -q -i -E "resource.*id=\"${INSTANCE_ATTR_NAME}\".*role=\"(Promoted|Master)\".*active=\"true\".*orphaned=\"false\".*failed=\"false\""
    return $?
}

clear_master_score()
{
    local node=$(ocf_attribute_target $1)
    if [ -z "$node" ]; then
        $CRM_MASTER -D
    else 
        $CRM_MASTER -D -N $node
    fi
}

set_master_score()
{
    local node=$(ocf_attribute_target $1)

    if [ -z "$node" ]; then
        $CRM_MASTER -v 100
    else 
        $CRM_MASTER -N $node -v 100
    fi
}

promote_everyone()
{

    for node in $(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' '); do
        local pcmk_node=$(galera_to_pcmk_name $node)
        if [ -z "$pcmk_node" ]; then
            ocf_log err "Could not determine pacemaker node from galera name <${node}>."
            return
        else
            node=$pcmk_node
        fi

        set_master_score $node
    done
}

greater_than_equal_long()
{
    # there are values we need to compare in this script
    # that are too large for shell -gt to process
    echo | awk -v n1="$1" -v n2="$2"  '{if (n1>=n2) printf ("true"); else printf ("false");}' |  grep -q "true"
}

galera_to_pcmk_name()
{
    local galera=$1
    if [ -z "$OCF_RESKEY_cluster_host_map" ]; then
        echo $galera
    else
        echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$2=="'"$galera"'" {print $1;exit}'
    fi
}

pcmk_to_galera_name()
{
    local pcmk=$1
    if [ -z "$OCF_RESKEY_cluster_host_map" ]; then
        echo $pcmk
    else
        echo "$OCF_RESKEY_cluster_host_map" | tr ';' '\n' | tr -d ' ' | sed 's/:/ /' | awk -F' ' '$1=="'"$pcmk"'" {print $2;exit}'
    fi
}


detect_first_master()
{
    local best_commit=0
    local last_commit=0
    local missing_nodes=0
    local nodes=""
    local nodes_recovered=""
    local all_nodes
    local best_node_gcomm
    local best_node
    local safe_to_bootstrap

    all_nodes=$(echo "$OCF_RESKEY_wsrep_cluster_address" | sed 's/gcomm:\/\///g' | tr -d ' ' | tr -s ',' ' ')
    best_node_gcomm=$(echo "$all_nodes" | sed 's/^.* \(.*\)$/\1/')
    best_node=$(galera_to_pcmk_name $best_node_gcomm)
    if [ -z "$best_node" ]; then
        ocf_log err "Could not determine initial best node from galera name <${best_node_gcomm}>."
        return
    fi

    # avoid selecting a recovered node as bootstrap if possible
    for node in $all_nodes; do
        local pcmk_node=$(galera_to_pcmk_name $node)
        if [ -z "$pcmk_node" ]; then
            ocf_log err "Could not determine pacemaker node from galera name <${node}>."
            return
        else
            node=$pcmk_node
        fi

        if is_no_grastate $node; then
            nodes_recovered="$nodes_recovered $node"
        else
            nodes="$nodes $node"
        fi
    done

    for node in $nodes_recovered $nodes; do
        # On clean shutdown, galera sets the last stopped node as 'safe to bootstrap',
        # so use this hint when we can
        safe_to_bootstrap=$(get_safe_to_bootstrap $node)

        # Special case for 2-node clusters: during a network split, rely on
        # pacemaker's quorum to check whether we can restart galera
        if [ "$safe_to_bootstrap" != "1" ] && [ "$node" = "$NODENAME" ] && is_two_node_mode_active; then
            is_last_node_in_quorate_partition
            if [ $? -eq 0 ]; then
                ocf_log warn "Survived a split in a 2-node cluster, considering ourselves safe to bootstrap"
                safe_to_bootstrap=1
            fi
        fi

        if [ "$safe_to_bootstrap" = "1" ]; then
            # Gale                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                