fix(gnovm): recover from preprocessing panics on node restart

[davd-gzl]

PreprocessAllFilesAndSaveBlockNodes now recovers per-package panics so a single broken package cannot crash the node on restart (e.g.: #5238). Failed packages are logged and skipped; gnoweb detects them via ListFiles and shows a dedicated 'Package Unavailable' page.

test_preprocess_recovery.sh:

#!/bin/bash
# test_preprocess_recovery.sh
#
# Verifies that a gnoland node survives restart when a package
# fails preprocessing. Uses a git worktree so the working tree
# stays clean, and a hand-crafted minimal genesis (no example
# packages) so startup is fast.
#
set -euxo pipefail

REPO_ROOT="$(cd "$(dirname "$0")" && pwd)"
WORK_DIR="$(mktemp -d)"
WORKTREE="$REPO_ROOT/test-worktree"
NODE_PID=""

log()  { echo -e "\033[0;32m[TEST]\033[0m $*"; }
fail() { echo -e "\033[0;31m[FAIL]\033[0m $*"; exit 1; }

# --- Pre-flight: ensure port 26657 is free ---
if curl -s http://127.0.0.1:26657/status >/dev/null 2>&1; then
    fail "Port 26657 already in use. Kill the existing node first."
fi

cleanup() {
    log "Cleanup..."
    [[ -n "$NODE_PID" ]] && kill "$NODE_PID" 2>/dev/null && wait "$NODE_PID" 2>/dev/null || true
    git -C "$REPO_ROOT" worktree remove --force "$WORKTREE" 2>/dev/null || true
    rm -rf "$WORK_DIR"
    log "Done."
}
trap cleanup EXIT

# --- Setup worktree from master ---
MASTER_HASH="$(git -C "$REPO_ROOT" rev-parse master)"
log "Creating worktree at $WORKTREE from master ($MASTER_HASH) ..."
git -C "$REPO_ROOT" worktree add "$WORKTREE" "$MASTER_HASH" --detach -q
log "Worktree created."

# Copy our modified sources (recovery logic) into the worktree.
log "Copying modified sources into worktree..."
cp "$REPO_ROOT/gnovm/pkg/gnolang/machine.go" "$WORKTREE/gnovm/pkg/gnolang/machine.go"
cp "$REPO_ROOT/gno.land/pkg/sdk/vm/keeper.go" "$WORKTREE/gno.land/pkg/sdk/vm/keeper.go"
log "Sources copied."

BIN="$WORK_DIR/bin"
NODE_DIR="$WORK_DIR/node"
KEYS="$WORK_DIR/keys"
REALM="$WORK_DIR/realm"
GENESIS="$WORK_DIR/genesis.json"
SECRETS="$NODE_DIR/secrets"
CONFIG="$NODE_DIR/config"
mkdir -p "$BIN" "$NODE_DIR" "$KEYS" "$REALM" "$SECRETS" "$CONFIG"

# --- Build ---
log "Building gnoland & gnokey..."
(cd "$WORKTREE/gno.land" && go build -o "$BIN/gnoland" ./cmd/gnoland)
(cd "$WORKTREE/gno.land" && go build -o "$BIN/gnokey"  ./cmd/gnokey)
log "Build done."

# --- Initialize node secrets and config ---
log "Initializing node secrets..."
"$BIN/gnoland" secrets init --data-dir "$SECRETS"
log "Initializing node config..."
"$BIN/gnoland" config init --config-path "$CONFIG/config.toml"

# Extract validator address and pubkey
VALIDATOR_ADDR=$(jq -r '.address' "$SECRETS/priv_validator_key.json")
PUBKEY_VALUE=$(jq -r '.pub_key.value' "$SECRETS/priv_validator_key.json")
log "Validator: $VALIDATOR_ADDR"

# --- Import test1 key ---
log "Importing test1 key..."
printf 'source bonus chronic canvas draft south burst lottery vacant surface solve popular case indicate oppose farm nothing bullet exhibit title speed wink action roast\n\n\n' \
    | "$BIN/gnokey" add test1 --home "$KEYS" --recover --insecure-password-stdin 2>&1 \
    || fail "Key import failed."
TEST1_ADDR="g1jg8mtutu9khhfwc4nxmuhcpftf0pajdhfvsqf5"
log "Key imported: $TEST1_ADDR"

# --- Create test realm ---
log "Creating test realm..."
cat > "$REALM/gnomod.toml" <<'TOML'
module = "gno.land/r/test/broken"
gno = "0.9"
TOML
cat > "$REALM/broken.gno" <<'GO'
package broken

func Render(path string) string {
	return "# Hello from the broken realm!"
}
GO
log "Realm created."

# --- Generate minimal genesis.json (NO example packages) ---
log "Generating minimal genesis.json..."
cat > "$GENESIS" <<GENESIS_EOF
{
  "genesis_time": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "chain_id": "test",
  "consensus_params": {
    "Block": {
      "MaxTxBytes": "1000000",
      "MaxDataBytes": "2000000",
      "MaxBlockBytes": "0",
      "MaxGas": "3000000000",
      "TimeIotaMS": "100"
    },
    "Validator": {
      "PubKeyTypeURLs": ["/tm.PubKeyEd25519"]
    }
  },
  "validators": [
    {
      "address": "$VALIDATOR_ADDR",
      "pub_key": {
        "@type": "/tm.PubKeyEd25519",
        "value": "$PUBKEY_VALUE"
      },
      "power": "10",
      "name": "testvalidator"
    }
  ],
  "app_hash": null,
  "app_state": {
    "@type": "/gno.GenesisState",
    "balances": [
      "${VALIDATOR_ADDR}=10000000000ugnot",
      "${TEST1_ADDR}=100000000000ugnot"
    ],
    "txs": [],
    "auth": {
      "params": {
        "max_memo_bytes": "65536",
        "tx_sig_limit": "7",
        "tx_size_cost_per_byte": "10",
        "sig_verify_cost_ed25519": "590",
        "sig_verify_cost_secp256k1": "1000",
        "gas_price_change_compressor": "10",
        "target_gas_ratio": "70",
        "initial_gasprice": {
          "gas": "1000",
          "price": "1ugnot"
        },
        "fee_collector": "g17xpfvakm2amg962yls6f84z3kell8c5lr9lr2e"
      }
    },
    "bank": {
      "params": {
        "restricted_denoms": []
      }
    },
    "vm": {
      "params": {
        "sysnames_pkgpath": "gno.land/r/sys/names",
        "syscla_pkgpath": "gno.land/r/sys/cla",
        "chain_domain": "gno.land",
        "default_deposit": "600000000ugnot",
        "storage_price": "100ugnot",
        "storage_fee_collector": "g1c9stkafpvcwez2efq3qtfuezw4zpaux3tvxggk"
      },
      "realm_params": null
    }
  }
}
GENESIS_EOF
log "Genesis created."

# --- Helper: wait for node RPC ---
wait_node() {
    for i in $(seq 1 120); do
        if curl -sf http://127.0.0.1:26657/status >/dev/null 2>&1; then
            return 0
        fi
        kill -0 "$NODE_PID" 2>/dev/null || return 1
        sleep 1
    done
    return 1
}

# Wait until node has produced at least one block
wait_block() {
    for i in $(seq 1 60); do
        HEIGHT=$(curl -sf http://127.0.0.1:26657/status 2>/dev/null \
            | jq -r '.result.sync_info.latest_block_height // "0"')
        if [[ "$HEIGHT" != "0" && "$HEIGHT" != "" ]]; then
            return 0
        fi
        sleep 1
    done
    return 1
}

# --- First boot: start node, deploy realm ---
log "Starting node (first boot)..."
"$BIN/gnoland" start \
    --data-dir "$NODE_DIR" --genesis "$GENESIS" --chainid test \
    --gnoroot-dir "$WORKTREE" \
    --skip-failing-genesis-txs --skip-genesis-sig-verification \
    --log-level error >"$WORK_DIR/boot1.log" 2>&1 &
NODE_PID=$!
log "Waiting for node (PID $NODE_PID)..."
wait_node || { tail -50 "$WORK_DIR/boot1.log"; fail "First boot failed."; }
log "Node RPC ready."
wait_block || fail "Node not producing blocks."
log "Node producing blocks."

log "Deploying r/test/broken ..."
printf '\n' | "$BIN/gnokey" maketx addpkg \
    --pkgdir "$REALM" --pkgpath gno.land/r/test/broken \
    --gas-fee 1000000ugnot --gas-wanted 100000000 \
    --broadcast --chainid test --remote 127.0.0.1:26657 \
    --home "$KEYS" --insecure-password-stdin test1 2>&1 \
    || fail "Deploy failed."
log "Realm deployed."

log "Stopping node..."
kill "$NODE_PID" && wait "$NODE_PID" 2>/dev/null || true
NODE_PID=""
sleep 1
log "Node stopped."

# --- Inject panic in worktree for our realm ---
log "Patching worktree: inject panic for r/test/broken ..."
sed -i '/mpkg = MPFProd.FilterMemPackage(mpkg)/i\
\t// INJECTED: simulate preprocessing failure\
\tif mpkg.Path == "gno.land/r/test/broken" {\
\t\tpanic("INJECTED: simulated preprocess crash")\
\t}\
' "$WORKTREE/gnovm/pkg/gnolang/machine.go"
log "Patch applied."

log "Rebuilding gnoland with injected panic..."
(cd "$WORKTREE/gno.land" && go build -o "$BIN/gnoland" ./cmd/gnoland)
log "Rebuild done."

# --- Second boot: should survive ---
log "Restarting node (second boot, with broken package)..."
"$BIN/gnoland" start \
    --data-dir "$NODE_DIR" --genesis "$GENESIS" --chainid test \
    --gnoroot-dir "$WORKTREE" \
    --log-level info >"$WORK_DIR/boot2.log" 2>&1 &
NODE_PID=$!
log "Waiting for node (PID $NODE_PID)..."
if ! wait_node; then
    echo "--- boot2.log (last 50 lines) ---"
    tail -50 "$WORK_DIR/boot2.log"
    fail "Node crashed on restart! Recovery did NOT work."
fi
log "Node restarted successfully."

# --- Verify warning logged ---
log "Checking logs for preprocessing failure warning..."
if grep -q "package preprocessing failed" "$WORK_DIR/boot2.log"; then
    log "PASS: Warning found in logs."
else
    fail "Expected 'package preprocessing failed' warning not found in logs."
fi

# --- Print boot2 log (the most important output) ---
log "=== boot2.log (restart with broken package) ==="
cat "$WORK_DIR/boot2.log"
log "=== end boot2.log ==="

log "Stopping node..."
kill "$NODE_PID" && wait "$NODE_PID" 2>/dev/null || true
NODE_PID=""

echo ""
log "ALL CHECKS PASSED"

[Gno2D2]

🛠 PR Checks Summary

🔴 Changes related to gnoweb must be reviewed by its codeowners
🔴 Pending initial approval by a review team member, or review from tech-staff

Manual Checks (for Reviewers):

• IGNORE the bot requirements for this PR (force green CI check)

Read More

🤖 This bot helps streamline PR reviews by verifying automated checks and providing guidance for contributors and reviewers.

✅ Automated Checks (for Contributors):

🟢 Maintainers must be able to edit this pull request (more info)
🔴 Changes related to gnoweb must be reviewed by its codeowners
🔴 Pending initial approval by a review team member, or review from tech-staff

☑️ Contributor Actions:

1. Fix any issues flagged by automated checks.
2. Follow the Contributor Checklist to ensure your PR is ready for review.
   • Add new tests, or document why they are unnecessary.
   • Provide clear examples/screenshots, if necessary.
   • Update documentation, if required.
   • Ensure no breaking changes, or include BREAKING CHANGE notes.
   • Link related issues/PRs, where applicable.

☑️ Reviewer Actions:

1. Complete manual checks for the PR, including the guidelines and additional checks if applicable.

📚 Resources:

• Report a bug with the bot.
• View the bot’s configuration file.

Debug

Automated Checks

Maintainers must be able to edit this pull request (more info)

If

🟢 Condition met
└── 🟢 And
    ├── 🟢 The base branch matches this pattern: ^master$
    └── 🟢 The pull request was created from a fork (head branch repo: davd-gzl/gno)

Then

🟢 Requirement satisfied
└── 🟢 Maintainer can modify this pull request

Changes related to gnoweb must be reviewed by its codeowners

If

🟢 Condition met
└── 🟢 And
    ├── 🟢 The base branch matches this pattern: ^master$
    └── 🟢 A changed file matches this pattern: ^gno.land/pkg/gnoweb/ (filename: gno.land/pkg/gnoweb/components/view_status.go)

Then

🔴 Requirement not satisfied
└── 🔴 Or
    ├── 🔴 Or
    │   ├── 🔴 And
    │   │   ├── 🔴 Pull request author is user: alexiscolin
    │   │   └── 🔴 This user reviewed pull request: gfanton (with state "APPROVED")
    │   └── 🔴 And
    │       ├── 🔴 Pull request author is user: gfanton
    │       └── 🔴 This user reviewed pull request: alexiscolin (with state "APPROVED")
    └── 🔴 And
        ├── 🟢 Not (🔴 Pull request author is user: alexiscolin)
        ├── 🟢 Not (🔴 Pull request author is user: gfanton)
        └── 🔴 Or
            ├── 🔴 This user reviewed pull request: alexiscolin (with state "APPROVED")
            └── 🔴 This user reviewed pull request: gfanton (with state "APPROVED")

Pending initial approval by a review team member, or review from tech-staff

If

🟢 Condition met
└── 🟢 And
    ├── 🟢 The base branch matches this pattern: ^master$
    └── 🟢 Not (🔴 Pull request author is a member of the team: tech-staff)

Then

🔴 Requirement not satisfied
└── 🔴 If
    ├── 🔴 Condition
    │   └── 🔴 Or
    │       ├── 🔴 At least one of these user(s) reviewed the pull request: [davd-gzl jefft0 notJoon omarsy MikaelVallenet] (with state "APPROVED")
    │       ├── 🔴 At least 1 user(s) of the team tech-staff reviewed pull request
    │       └── 🔴 This pull request is a draft
    └── 🔴 Else
        └── 🔴 And
            ├── 🟢 This label is applied to pull request: review/triage-pending
            └── 🔴 On no pull request

Manual Checks

**IGNORE** the bot requirements for this PR (force green CI check)

If

🟢 Condition met
└── 🟢 On every pull request

Can be checked by

• Any user with comment edit permission

[codecov]

Codecov Report

❌ Patch coverage is 46.66667% with 24 lines in your changes missing coverage. Please review.

Files with missing lines	Patch %	Lines
gno.land/pkg/gnoweb/components/view_status.go	0.00%	11 Missing ⚠️
gno.land/pkg/gnoweb/handler_http.go	0.00%	7 Missing and 1 partial ⚠️
gno.land/pkg/sdk/vm/keeper.go	50.00%	2 Missing and 1 partial ⚠️
gnovm/pkg/gnolang/machine.go	90.00%	1 Missing and 1 partial ⚠️

📢 Thoughts on this report? Let us know!

[notJoon]

an lint error has occurred. could you please take a look? thank you!

[davd-gzl]

fix: 40a2a82 @notJoon