#!/usr/bin/env bash set -e # Invoke with hevm e.g. # hevm compliance --tests ~/ethereum-tests --skip modexp --timeout 20 --html HEVM=${HEVM:-hevm} if [[ "$#" -lt 1 ]]; then echo >&2 "usage: $(basename "$0") " exit 1 fi tests=$1 html=$2 match=$3 skip=$4 timeout=${5:-10} _html () { cat <<. hevm test results

hevm consensus test report

$(date +%Y-%m-%d)

$(echo "$npass passed, $nbal bad-balance, $nnon bad-nonce, $nstr bad-storage, $nfail failed, $nskip skipped, $ntime timeout")

(Test suite: GeneralStateTests for Istanbul)

Failed tests

$(echo $noncefailed) $(echo $storagefailed) $(echo $failed)

Failed tests (due to balance only)

$(echo $balancefailed)

Timeout tests

$(echo $timeouts)

Skipped tests

$(echo $skipped)

Passed tests

$(echo $passed)
. } shopt -s nocasematch { cd "$tests" for x in BlockchainTests/GeneralStateTests/*/*; do if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then for job in $(<$x jq '.|keys[]' -r); do echo -n "$job " ; echo "skip" done elif [[ $x =~ .*$match.* ]]; then set +e "$HEVM" bc-test --file $x --timeout $timeout 2>&1 set -e fi done } | { while read test outcome; do echo >&2 "$test $outcome" row="$test$outcome" row+=$'\n' case $outcome in ok) passed+=$row ;; bad-balance) balancefailed+=$row ;; bad-nonce) noncefailed+=$row ;; bad-storage) storagefailed+=$row ;; timeout) timeouts+=$row ;; skip) skipped+=$row ;; *) failed+=$row ;; esac done sum () { echo -ne "$1" | wc -l | awk '{print $1}'; } npass=$(sum "$passed") nbal=$(sum "$balancefailed") nnon=$(sum "$noncefailed") nstr=$(sum "$storagefailed") nfail=$(sum "$failed") ntime=$(sum "$timeouts") nskip=$(sum "$skipped") echo >&2 "passed: $npass" echo >&2 "bad-balance: $nbal" echo >&2 "bad-nonce: $nnon" echo >&2 "bad-storage: $nstr" echo >&2 "failed: $nfail" echo >&2 "timeout: $ntime" echo >&2 "skipped: $nskip" if [[ $html == "True" ]]; then _html fi nbad=$(($nbal + $nnon + $nstr + $nfail)) [[ $nbad -gt 0 ]] && exit 1 || exit 0 }