#!/usr/bin/env bash # Integration test for Notary admin rescue commands # This script tests the complete lifecycle of Notary rescue operations: # 1. Create synthetic delayed/blackholed messages using inject commands # 2. Execute admin commands against the guardian # 3. Verify state changes after each command # # This modifies Tilt's BadgerDB, so successive runs of this script # will create many stored messages and the DB is not cleared after the # script terminates. # # Prerequisites: # - Tilt devnet must be running with notary enabled # # Usage: ./test-notary-admin-commands.sh [NODE_NUMBER] # NODE_NUMBER: Guardian node to test against (default: 0) set -e node=${1:-0} # Default to guardian-0 sock="/tmp/admin.sock" echo "=== Notary Admin Commands Integration Test ===" echo "Testing against guardian-${node}" echo "" # Helper function to execute commands in guardian pod exec_in_pod() { kubectl exec -n wormhole guardian-${node} -c guardiand -- "$@" } # Helper function to run admin commands admin_cmd() { exec_in_pod /guardiand admin "$@" --socket "$sock" } # Helper function to print database state print_db_state() { local title="$1" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "📊 $title" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" local delayed_output=$(admin_cmd notary-list-delayed-messages) local delayed_count=$(echo "$delayed_output" | grep -oP 'Delayed messages \(\K[0-9]+' || echo "0") echo "🕒 DELAYED MESSAGES: $delayed_count" if [[ "$delayed_count" -gt 0 ]]; then echo "$delayed_output" | grep "^2/" | while read -r msg_id; do local details=$(admin_cmd notary-get-delayed-message "$msg_id" 2>/dev/null || echo "") local release_time=$(echo "$details" | grep -oP 'Release Time: \K.*' || echo "unknown") echo " • ${msg_id:0:80}..." echo " Release: $release_time" done fi echo "" local blackholed_output=$(admin_cmd notary-list-blackholed-messages) local blackholed_count=$(echo "$blackholed_output" | grep -oP 'Blackholed messages \(\K[0-9]+' || echo "0") echo "⛔ BLACKHOLED MESSAGES: $blackholed_count" if [[ "$blackholed_count" -gt 0 ]]; then echo "$blackholed_output" | grep "^2/" | while read -r msg_id; do echo " • ${msg_id:0:80}..." done fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" } # Step 1: Create synthetic test state using inject commands echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 1: Creating synthetic delayed messages" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" msg1_output=$(admin_cmd notary-inject-delayed-message 4) msg1_id=$(echo "$msg1_output" | grep -oP 'Injected delayed message: \K.*') msg2_output=$(admin_cmd notary-inject-delayed-message 4) msg2_id=$(echo "$msg2_output" | grep -oP 'Injected delayed message: \K.*') echo "✓ Created test messages:" echo " - Message 1 (4 day delay): ${msg1_id:0:80}..." echo " - Message 2 (4 day delay): ${msg2_id:0:80}..." print_db_state "Database State After Injection" # Step 2: Verify initial state using list command delayed_list=$(admin_cmd notary-list-delayed-messages) if ! echo "$delayed_list" | grep -q "$msg1_id"; then echo "✗ FAIL: Message 1 not found in delayed list" exit 1 fi if ! echo "$delayed_list" | grep -q "$msg2_id"; then echo "✗ FAIL: Message 2 not found in delayed list" exit 1 fi echo "✓ Initial state verified" # Step 3: Test notary-blackhole-delayed-message echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 2: Testing notary-blackhole-delayed-message" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Moving message 1 from delayed → blackholed..." admin_cmd notary-blackhole-delayed-message "$msg1_id" # Give DB a moment to flush the write sleep 0.5 delayed_list=$(admin_cmd notary-list-delayed-messages) if echo "$delayed_list" | grep -q "$msg1_id"; then echo "✗ FAIL: Message still in delayed list after blackhole" exit 1 fi blackholed_list=$(admin_cmd notary-list-blackholed-messages) if ! echo "$blackholed_list" | grep -q "$msg1_id"; then echo "✗ FAIL: Message not moved to blackholed list" exit 1 fi echo "✓ Message successfully moved from delayed → blackholed" print_db_state "Database State After Blackholing" # Step 4: Test notary-remove-blackholed-message echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 3: Testing notary-remove-blackholed-message" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Removing message 1 from blackholed (restoring to delayed)..." admin_cmd notary-remove-blackholed-message "$msg1_id" # Give DB a moment to flush the write sleep 0.5 blackholed_list=$(admin_cmd notary-list-blackholed-messages) if echo "$blackholed_list" | grep -q "$msg1_id"; then echo "✗ FAIL: Message still in blackholed list after removal" exit 1 fi delayed_list=$(admin_cmd notary-list-delayed-messages) if ! echo "$delayed_list" | grep -q "$msg1_id"; then echo "✗ FAIL: Message not restored to delayed list" exit 1 fi echo "✓ Message successfully moved from blackholed → delayed" print_db_state "Database State After Removing from Blackhole" # Step 5: Test notary-release-delayed-message echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 4: Testing notary-release-delayed-message" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Getting release time BEFORE release command..." before_release=$(admin_cmd notary-get-delayed-message "$msg2_id") before_time=$(echo "$before_release" | grep -oP 'Release Time: \K.*') echo " Before: $before_time" echo "" echo "Releasing message 2 (sets release time to now)..." admin_cmd notary-release-delayed-message "$msg2_id" # Note: The notary processes delayed messages on a 1-minute poll timer. # The release sets the time in the past, but we need to wait for the next poll cycle. # For testing purposes, we'll just verify the message is still in the delayed list # but with a release time in the past. delayed_msg_info=$(admin_cmd notary-get-delayed-message "$msg2_id") if ! echo "$delayed_msg_info" | grep -q "$msg2_id"; then echo "✗ FAIL: Message not found after release command" exit 1 fi after_time=$(echo "$delayed_msg_info" | grep -oP 'Release Time: \K.*') echo " After: $after_time" echo "" echo "✓ Release time updated (will be released on next poll cycle in ~60s)" print_db_state "Database State After Release (time updated)" # Step 6: Test notary-reset-release-timer echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 5: Testing notary-reset-release-timer" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" msg3_output=$(admin_cmd notary-inject-delayed-message 1) msg3_id=$(echo "$msg3_output" | grep -oP 'Injected delayed message: \K.*') echo "Created message 3 with 1 day delay: ${msg3_id:0:80}..." echo "" before_reset=$(admin_cmd notary-get-delayed-message "$msg3_id") before_time=$(echo "$before_reset" | grep -oP 'Release Time: \K.*') echo "Getting release time BEFORE reset..." echo " Before: $before_time" echo "" echo "Resetting timer to 5 days..." admin_cmd notary-reset-release-timer "$msg3_id" 5 delayed_list=$(admin_cmd notary-list-delayed-messages) if ! echo "$delayed_list" | grep -q "$msg3_id"; then echo "✗ FAIL: Message not found after timer reset" exit 1 fi after_reset=$(admin_cmd notary-get-delayed-message "$msg3_id") after_time=$(echo "$after_reset" | grep -oP 'Release Time: \K.*') echo " After: $after_time" echo "" echo "✓ Release timer successfully reset from 1 day → 5 days" print_db_state "Database State After Timer Reset" # Step 7: Test introspection commands echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Step 6: Testing introspection commands" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" msg4_output=$(admin_cmd notary-inject-delayed-message 2) msg4_id=$(echo "$msg4_output" | grep -oP 'Injected delayed message: \K.*') msg5_output=$(admin_cmd notary-inject-blackholed-message) msg5_id=$(echo "$msg5_output" | grep -oP 'Injected blackholed message: \K.*') echo "Testing notary-get-delayed-message for message 4..." delayed_details=$(admin_cmd notary-get-delayed-message "$msg4_id") if ! echo "$delayed_details" | grep -q "$msg4_id"; then echo "✗ FAIL: Failed to get delayed message details" exit 1 fi echo "$delayed_details" echo "" echo "Testing notary-get-blackholed-message for message 5..." blackholed_details=$(admin_cmd notary-get-blackholed-message "$msg5_id") if ! echo "$blackholed_details" | grep -q "$msg5_id"; then echo "✗ FAIL: Failed to get blackholed message details" exit 1 fi echo "$blackholed_details" print_db_state "Final Database State"