#!/usr/bin/env bash set -e # Invoke with hevm e.g. # hevm compliance --tests ~/ethereum-tests --group VM --skip quadratic --html HEVM=${HEVM:-hevm} if [[ "$#" -lt 1 ]]; then echo >&2 "usage: $(basename "$0") " exit 1 fi tests=$1 html=$2 match=$3 skip=$4 timeout=${5:-10} _html() { cat <<. hevm test results

hevm consensus test report

$(date +%Y-%m-%d)

$(echo "$npass passed, $nfail failed, $nskip skipped")

(Test suite: VMTests for ConstantinopleFix)

Failed tests

$(echo $failed)

Skipped tests

$(echo $skipped)

Passed tests

$(echo $passed)
. } { cd "$tests" for x in VMTests/*/*; do if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then for job in $(<$x jq '.|keys[]' -r); do echo "$x $job skip" done elif [[ $x =~ .*$match.* ]]; then echo -n "$x " ; "$HEVM" vm-test --file $x --timeout $timeout 2>&1 fi done } | { while read path test outcome; do echo >&2 "$path $test $outcome" category=$(dirname "$path") testcase=$(basename "${path%.json}") row="$testcase$outcome$category" row+=$'\n' case $outcome in ok) passed+=$row ;; skip) skipped+=$row ;; timeout) timouts+=row ;; *) failed+=$row ;; esac done sum () { echo -ne "$1" | wc -l | awk '{print $1}'; } nfail=$(sum "$failed") npass=$(sum "$passed") nskip=$(sum "$skipped") ntime=$(sum "$timeouts") echo >&2 "passed: $npass" echo >&2 "failed: $nfail" echo >&2 "timeout: $ntime" echo >&2 "skipped: $nskip" if [[ $html == "True" ]]; then _html fi [[ $nfail -gt 0 ]] && exit 1 || exit 0 }