test_failover.sh

#!/bin/bash
# test_failover.sh

echo "=== Manual Failover Test ==="
echo "WARNING: This will trigger a failover!"
read -p "Continue? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
    exit 1
fi

DB1_IP="10.116.238.92"
DB2_IP="10.116.238.25"
DB3_IP="10.116.238.228"

# Find current master
current_master=""
for ip in "$DB1_IP" "$DB2_IP" "$DB3_IP"; do
    role=$(PGPASSWORD=postgres psql -h "$ip" -U postgres -d postgres -t -c "SELECT CASE WHEN pg_is_in_recovery() THEN 'replica' ELSE 'master' END;" 2>/dev/null | tr -d '[:space:]')
    if [ "$role" = "master" ]; then
        current_master="$ip"
        break
    fi
done

echo "Current master: $current_master"

# Create test data
echo "Creating test data before failover..."
PGPASSWORD=postgres psql -h "$current_master" -U postgres -d postgres -c "
    CREATE TABLE IF NOT EXISTS failover_test (
        id SERIAL PRIMARY KEY,
        data TEXT,
        created_at TIMESTAMP DEFAULT NOW()
    );
    INSERT INTO failover_test (data) VALUES ('Before failover $(date)');
"

# Get target for switchover (not current master)
target=""
for ip in "$DB1_IP" "$DB2_IP" "$DB3_IP"; do
    if [ "$ip" != "$current_master" ]; then
        target="$ip"
        break
    fi
done

# Trigger failover
echo "Triggering failover to $target..."
ssh postgres@"$current_master" "patronictl -c /etc/patroni/patroni.yml switchover --master $current_master --candidate $(echo $target | sed 's/.*\.//') --force" || echo "Failover command executed"

# Wait and check
echo "Waiting for failover to complete..."
sleep 15

# Find new master
new_master=""
for ip in "$DB1_IP" "$DB2_IP" "$DB3_IP"; do
    role=$(PGPASSWORD=postgres psql -h "$ip" -U postgres -d postgres -t -c "SELECT CASE WHEN pg_is_in_recovery() THEN 'replica' ELSE 'master' END;" 2>/dev/null | tr -d '[:space:]')
    if [ "$role" = "master" ]; then
        new_master="$ip"
        break
    fi
done

echo "New master: $new_master"

# Test write to new master
echo "Testing write to new master..."
PGPASSWORD=postgres psql -h "$new_master" -U postgres -d postgres -c "
    INSERT INTO failover_test (data) VALUES ('After failover $(date)');
    SELECT * FROM failover_test ORDER BY id;
"

echo "Failover test completed!"