From 4aa565f59df6da0ad1106558baadc918713ecbf2 Mon Sep 17 00:00:00 2001
From: Martin Pitt <mpitt@redhat.com>
Date: Fri, 5 Mar 2021 09:36:15 +0100
Subject: [PATCH] test: Add FMF test metadata and scripts

Copy and adjust the browser.sh and run-test.sh scripts from
Fedora's downstream dist-git gating tests [1], and translate the STI
yaml to an FMF metadata file test/verify.fmf. This replaces the previous
test/main.fmf, which was not functional and just a stub.

With this we can share the same test scripts upstream (for testing in
Packit) and downstream.

[1] https://src.fedoraproject.org/rpms/cockpit/blob/71ee0da592bd10d8dde9d9271e3e2c62ab93a6bb/f/tests
---
 {test/.fmf => .fmf}/version  |   0
 plans/all.fmf                |   6 ++
 test/main.fmf                |   6 --
 test/run-verify-host-user.sh | 116 +++++++++++++++++++++++++++++++++++
 test/run-verify-host.sh      |  60 ++++++++++++++++++
 test/verify.fmf              |  23 +++++++
 tools/test-static-code       |   2 +-
 7 files changed, 206 insertions(+), 7 deletions(-)
 rename {test/.fmf => .fmf}/version (100%)
 create mode 100644 plans/all.fmf
 delete mode 100644 test/main.fmf
 create mode 100755 test/run-verify-host-user.sh
 create mode 100755 test/run-verify-host.sh
 create mode 100644 test/verify.fmf

diff --git a/test/.fmf/version b/.fmf/version
similarity index 100%
rename from test/.fmf/version
rename to .fmf/version
diff --git a/plans/all.fmf b/plans/all.fmf
new file mode 100644
index 0000000000..2bce34a64a
--- /dev/null
+++ b/plans/all.fmf
@@ -0,0 +1,6 @@
+summary:
+    Run all tests
+discover:
+    how: fmf
+execute:
+    how: tmt
diff --git a/test/main.fmf b/test/main.fmf
deleted file mode 100644
index 4622bd50ef..0000000000
--- a/test/main.fmf
+++ /dev/null
@@ -1,6 +0,0 @@
-duration: 2h
-test: common/run-tests
-environment:
-  TEST_DIR: verify
-  #TRACE: true
-  #SIT: True
diff --git a/test/run-verify-host-user.sh b/test/run-verify-host-user.sh
new file mode 100755
index 0000000000..d59f1d36be
--- /dev/null
+++ b/test/run-verify-host-user.sh
@@ -0,0 +1,116 @@
+#!/bin/sh
+set -eux
+
+cd "$SOURCE"
+
+. /etc/os-release
+test_optional=
+test_basic=
+
+if ls ../cockpit-appstream* 1> /dev/null 2>&1; then
+    test_optional=1
+else
+    test_basic=1
+fi
+
+if [ "$ID" = "fedora" ]; then
+    test_basic=1
+    test_optional=1
+fi
+
+# tests need cockpit's bots/ libraries
+git clone --depth=1 https://github.com/cockpit-project/bots
+
+# support running from clean git tree
+if [ ! -d node_modules/chrome-remote-interface ]; then
+    npm install chrome-remote-interface sizzle
+fi
+
+export TEST_OS="${ID}-${VERSION_ID/./-}"
+# HACK: upstream does not yet know about rawhide
+if [ "$TEST_OS" = "fedora-35" ]; then
+    export TEST_OS=fedora-33
+fi
+
+# HACK: CI hits this selinux denial. Unrelated to our tests.
+export TEST_ALLOW_JOURNAL_MESSAGES=".*Permission denied:.*/var/cache/app-info/xmls.*"
+
+# select tests
+TESTS=""
+EXCLUDES=""
+RC=0
+if [ -n "$test_optional" ]; then
+    # pre-download cirros image for Machines tests
+    bots/image-download cirros
+
+    # triggers SELinux violation
+    # See journal: SELinux is preventing /usr/libexec/qemu-kvm from open access on the file /var/lib/cockpittest/nfs_pool/nfs-volume-0.
+    EXCLUDES="$EXCLUDES TestMachinesDisks.testAddDiskNFS"
+    # not investigated yet
+    EXCLUDES="$EXCLUDES
+        TestAutoUpdates.testPrivilegeChange"
+
+    # TestUpdates: we can't run rebooting tests
+    TESTS="$TESTS
+         TestAutoUpdates
+         TestStorage
+         TestUpdates.testBasic
+         TestUpdates.testSecurityOnly"
+
+    # Fedora gating tests are running on infra without /dev/kvm; Machines tests are too darn slow there
+    if [ "$ID" = "fedora" ]; then
+        TESTS="$TESTS TestMachinesCreate.testCreateImportDisk"
+    else
+        TESTS="$TESTS TestMachines"
+    fi
+fi
+
+if [ -n "$test_basic" ]; then
+    # still too unstable
+    EXCLUDES="$EXCLUDES TestFirewall.testNetworkingPage"
+
+    # TODO: fix for CI environment
+    EXCLUDES="$EXCLUDES TestLogin.testTally"
+    EXCLUDES="$EXCLUDES TestAccounts.testBasic"
+
+    # PCI devices list is not predictable
+    EXCLUDES="$EXCLUDES TestSystemInfo.testHardwareInfo"
+
+    # Known issue #1008
+    EXCLUDES="$EXCLUDES TestTuned.testBasic"
+
+    TESTS="$TESTS
+        TestAccounts
+        TestBonding
+        TestBridge
+        TestFirewall
+        TestKdump
+        TestLogin
+        TestNetworking
+        TestServices
+        TestSOS
+        TestSystemInfo
+        TestTeam
+        TestTerminal
+        TestTuned
+        "
+fi
+
+exclude_options=""
+for t in $EXCLUDES; do
+    exclude_options="$exclude_options --exclude $t"
+done
+
+# execute run-tests
+test/common/run-tests --test-dir test/verify --nondestructive $exclude_options \
+    --machine localhost:22 --browser localhost:9090 $TESTS || RC=$?
+
+# check-shell-menu is not @nondestructive yet, keep it last
+if [ -n "$test_basic" ]; then
+    test/verify/check-shell-menu --machine localhost:22 --browser localhost:9090 || RC=$?
+fi
+
+echo $RC > "$LOGS/exitcode"
+cp --verbose Test* "$LOGS" || true
+# deliver test result via exitcode file
+exit 0
diff --git a/test/run-verify-host.sh b/test/run-verify-host.sh
new file mode 100755
index 0000000000..4799081775
--- /dev/null
+++ b/test/run-verify-host.sh
@@ -0,0 +1,60 @@
+#!/bin/sh
+# This script is meant to be run on an ephemeral CI host, for packit/Fedora/RHEL gating.
+set -eux
+
+TESTS="$(realpath $(dirname "$0"))"
+if [ -d source ]; then
+    # path for standard-test-source
+    SOURCE="$(pwd)/source"
+else
+    SOURCE="$(realpath $TESTS/..)"
+fi
+LOGS="$(pwd)/logs"
+mkdir -p "$LOGS"
+chmod a+w "$LOGS"
+
+# install browser; on RHEL, use chromium from epel
+# HACK: chromium-headless ought to be enough, but version 88 has a crash: https://bugs.chromium.org/p/chromium/issues/detail?id=1170634
+if ! rpm -q chromium; then
+    if grep -q 'ID=.*rhel' /etc/os-release; then
+        dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+        dnf config-manager --enable epel
+    fi
+    dnf install -y chromium
+fi
+
+# make libpwquality less aggressive, so that our "foobar" password works
+printf 'dictcheck = 0\nminlen = 6\n' >> /etc/security/pwquality.conf
+
+# set root password for logging in
+echo root:foobar | chpasswd
+
+# create user account for logging in
+if ! id admin 2>/dev/null; then
+    useradd -c Administrator -G wheel admin
+    echo admin:foobar | chpasswd
+fi
+
+# create user account for running the test
+if ! id runtest 2>/dev/null; then
+    useradd -c 'Test runner' runtest
+    # allow test to set up things on the machine
+    mkdir -p /root/.ssh
+    curl https://raw.githubusercontent.com/cockpit-project/bots/master/machine/identity.pub  >> /root/.ssh/authorized_keys
+    chmod 600 /root/.ssh/authorized_keys
+fi
+chown -R runtest "$SOURCE"
+
+# disable core dumps, we rather investigate them upstream where test VMs are accessible
+echo core > /proc/sys/kernel/core_pattern
+
+# make sure that we can access cockpit through the firewall
+systemctl start firewalld
+firewall-cmd --add-service=cockpit --permanent
+firewall-cmd --add-service=cockpit
+
+# Run tests as unprivileged user
+su - -c "env SOURCE=$SOURCE LOGS=$LOGS $TESTS/run-verify-host-user.sh" runtest
+
+RC=$(cat $LOGS/exitcode)
+exit ${RC:-1}
diff --git a/test/verify.fmf b/test/verify.fmf
new file mode 100644
index 0000000000..f8dbdea902
--- /dev/null
+++ b/test/verify.fmf
@@ -0,0 +1,23 @@
+summary: Run browser integration tests on the host
+require:
+  - cockpit
+  - cockpit-tests
+  - cockpit-machines
+  - cockpit-sosreport
+  - cockpit-storaged
+  - createrepo_c
+  - cryptsetup
+  - dnf-automatic
+  - firewalld
+  - git
+  - libvirt-daemon-config-network
+  - libvirt-python3
+  - make
+  - NetworkManager-team
+  - npm
+  - python3
+  - sssd-dbus
+  - targetcli
+  - tlog
+test: ./run-verify-host.sh
+duration: 2h
diff --git a/tools/test-static-code b/tools/test-static-code
index dc942a57f8..af504ef66f 100755
--- a/tools/test-static-code
+++ b/tools/test-static-code
@@ -120,7 +120,7 @@ if [ -z "${PYTHON_STYLE_CHECKER-}" ]; then
     echo "ok 7 pycodestyle test # SKIP pycodestyle not installed"
 else
     # FIXME: Fix code for the warnings and re-enable them
-    out=$(python3 -m "$PYTHON_STYLE_CHECKER" --ignore E501,E265,E261,W504,W605 test/* --exclude=test/verify/nested-kvm,test/README.md,test/run,test/main.fmf) || true
+    out=$(python3 -m "$PYTHON_STYLE_CHECKER" --ignore E501,E265,E261,W504,W605 test/* --exclude=test/verify/nested-kvm,test/README.md,test/run,test/verify.fmf,test/run-verify-host.sh,test/run-verify-host-user.sh) || true
     if [ -n "$out" ]; then
         echo "$out" >&2
         echo "not ok 7 $PYTHON_STYLE_CHECKER test"