pytest: add tests using sshd

With either /usr/sbin/sshd found or configured via --with-test-sshd=path
add tests for SCP down- and uploads, insecure, with known hosts or not,
with authorized user key or unauthorized one.

Working now with libssh and libssh2, using a hashed known_hosts file.

Closes #19934

configure.ac

@@ -409,6 +409,51 @@ if test "$DANTED_ENABLED" = "no"; then
 fi
 AC_SUBST(DANTED)
 
+dnl we would like a sshd as test server
+dnl
+SSHD_ENABLED="maybe"
+AC_ARG_WITH(test-sshd, [AS_HELP_STRING([--with-test-sshd=PATH],
+                         [where to find sshd for testing])],
+  [request_sshd=$withval], [request_sshd=check])
+if test "x$request_sshd" = "xcheck" || test "x$request_sshd" = "xyes"; then
+  if test -x "/usr/sbin/sshd"; then
+    # common location on distros (debian/ubuntu)
+    SSHD="/usr/sbin/sshd"
+  else
+    AC_PATH_PROG([SSHD], [sshd])
+    if test -z "$SSHD"; then
+      AC_PATH_PROG([SSHD], [sshd])
+    fi
+  fi
+elif test "x$request_sshd" != "xno"; then
+  SSHD="${request_sshd}"
+  if test ! -x "${SSHD}"; then
+    AC_MSG_NOTICE([sshd not found as ${SSHD}, sshd tests disabled])
+    SSHD_ENABLED="no"
+  else
+    AC_MSG_NOTICE([using SSHD=$SSHD for tests])
+  fi
+fi
+if test "$SSHD_ENABLED" = "no"; then
+  SSHD=""
+  SFTPD=""
+else
+  if test -x "/usr/libexec/sftp-server"; then
+    # common location on macOS)
+    SFTPD="/usr/libexec/sftp-server"
+  elif test -x "/usr/lib/openssh/sftp-server"; then
+    # common location on debian
+    SFTPD="/usr/lib/openssh/sftp-server"
+  else
+    AC_PATH_PROG([SFTPD], [sftp-server])
+    if test -z "$SFTPD"; then
+      AC_PATH_PROG([SFTPD], [sftp-server])
+    fi
+  fi
+fi
+AC_SUBST(SSHD)
+AC_SUBST(SFTPD)
+
 dnl the nghttpx we might use in httpd testing
 if test -n "$TEST_NGHTTPX" && test "x$TEST_NGHTTPX" != "xnghttpx"; then
   HTTPD_NGHTTPX="$TEST_NGHTTPX"

docs/INSTALL-CMAKE.md

@@ -483,6 +483,8 @@ Examples:
 - `DANTED`:                                 Default: `danted`
 - `TEST_NGHTTPX`:                           Default: `nghttpx`
 - `VSFTPD`:                                 Default: `vsftps`
+- `SSHD`:                                   Default: `sshd`
+- `SFTPD`:                                  Default: `sftp-server`
 
 ## Feature detection variables
 

tests/http/CMakeLists.txt

@@ -58,5 +58,17 @@ if(NOT DANTED)
 endif()
 mark_as_advanced(DANTED)
 
-# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD
+find_program(SSHD "sshd" PATHS "/usr/sbin" "/usr/bin")
+if(NOT SSHD)
+  set(SSHD "")
+endif()
+mark_as_advanced(SSHD)
+
+find_program(SFTPD "sftp-server" PATHS "/usr/libexec" "/usr/lib/openssh")
+if(NOT SFTPD)
+  set(SFTPD "")
+endif()
+mark_as_advanced(SFTPD)
+
+# Consumed variables: APXS, CADDY, HTTPD, HTTPD_NGHTTPX, DANTED, VSFTPD, SSHD
 configure_file("config.ini.in" "${CMAKE_CURRENT_BINARY_DIR}/config.ini" @ONLY)

tests/http/Makefile.am

@@ -34,6 +34,7 @@ testenv/httpd.py                      \
 testenv/mod_curltest/mod_curltest.c   \
 testenv/nghttpx.py                    \
 testenv/ports.py                      \
+testenv/sshd.py                       \
 testenv/vsftpd.py                     \
 testenv/ws_echo_server.py
 
@@ -65,7 +66,9 @@ test_20_websockets.py  \
 test_30_vsftpd.py      \
 test_31_vsftpds.py     \
 test_32_ftps_vsftpd.py \
-test_40_socks.py \
+test_40_socks.py       \
+test_50_scp.py         \
+test_51_sftp.py        \
 $(TESTENV)
 
 clean-local:

tests/http/config.ini.in

@@ -40,3 +40,7 @@ vsftpd = @VSFTPD@
 
 [danted]
 danted = @DANTED@
+
+[sshd]
+sshd = @SSHD@
+sftpd = @SFTPD@

tests/http/conftest.py

@@ -34,7 +34,7 @@ from testenv.env import EnvConfig
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '.'))
 
-from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd
+from testenv import Env, Nghttpx, Httpd, NghttpxQuic, NghttpxFwd, Sshd
 
 log = logging.getLogger(__name__)
 
@@ -134,6 +134,17 @@ def nghttpx_fwd(env, httpd) -> Generator[Union[Nghttpx,bool], None, None]:
         yield False
 
 
+@pytest.fixture(scope='session')
+def sshd(env: Env) -> Generator[Union[Sshd,bool], None, None]:
+    if env.has_sshd():
+        sshd = Sshd(env=env)
+        assert sshd.initial_start(), f'{sshd.dump_log()}'
+        yield sshd
+        sshd.stop()
+    else:
+        yield False
+
+
 @pytest.fixture(scope='session')
 def configures_httpd(env, httpd) -> Generator[bool, None, None]:
     # include this fixture as test parameter if the test configures httpd itself

tests/http/test_50_scp.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import difflib
+import filecmp
+import logging
+import os
+import pytest
+
+from testenv import Env, CurlClient, Sshd
+
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(condition=not Env.curl_has_protocol('scp'), reason="curl built without scp:")
+@pytest.mark.skipif(condition=not Env.has_sshd(), reason="missing sshd")
+class TestScp:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env, sshd):
+        env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024)
+        env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024)
+        env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024)
+        env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024)
+
+    def test_50_01_insecure(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--insecure',
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+
+    def test_50_02_unknown_hosts(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.unknown_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(60)  # CURLE_PEER_FAILED_VERIFICATION
+
+    def test_50_03_known_hosts(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+
+    # use key not in authorized_keys file
+    def test_50_04_unauth_user(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user2_pubkey_file,
+            '--key', sshd.user2_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(67)  # CURLE_LOGIN_DENIED
+
+    def test_50_10_dl_single(self, env: Env, sshd: Sshd):
+        count = 1
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_50_11_dl_serial(self, env: Env, sshd: Sshd):
+        count = 5
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_50_12_dl_parallel(self, env: Env, sshd: Sshd):
+        count = 5
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'scp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.http_download(urls=[url], extra_args=[
+            '--parallel',
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_50_20_ul_single(self, env: Env, sshd: Sshd):
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_20.data')
+        curl = CurlClient(env=env)
+        url = f'scp://{env.domain1}:{sshd.port}/{destfile}'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_upload(sshd, srcfile, destfile)
+
+    def test_50_21_ul_serial(self, env: Env, sshd: Sshd):
+        count = 5
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_21.data')
+        curl = CurlClient(env=env)
+        url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        for i in range(count):
+            self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+    def test_50_22_ul_parallel(self, env: Env, sshd: Sshd):
+        count = 5
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_22.data')
+        curl = CurlClient(env=env)
+        url = f'scp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--parallel',
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        for i in range(count):
+            self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+    def check_downloads(self, client, srcfile: str, count: int,
+                        complete: bool = True):
+        for i in range(count):
+            dfile = client.download_file(i)
+            assert os.path.exists(dfile)
+            if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
+                diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+                                                    b=open(dfile).readlines(),
+                                                    fromfile=srcfile,
+                                                    tofile=dfile,
+                                                    n=1))
+                assert False, f'download {dfile} differs:\n{diff}'
+
+    def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True):
+        assert os.path.exists(srcfile)
+        assert os.path.exists(destfile)
+        if not filecmp.cmp(srcfile, destfile, shallow=False):
+            diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+                                                b=open(destfile).readlines(),
+                                                fromfile=srcfile,
+                                                tofile=destfile,
+                                                n=1))
+            assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}'

tests/http/test_51_sftp.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import difflib
+import filecmp
+import logging
+import os
+import pytest
+
+from testenv import Env, CurlClient, Sshd
+
+
+log = logging.getLogger(__name__)
+
+
+@pytest.mark.skipif(condition=not Env.curl_has_protocol('sftp'), reason="curl built without sfto:")
+@pytest.mark.skipif(condition=not Env.has_sftpd(), reason="missing sftp server")
+class TestSftp:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env, sshd):
+        env.make_data_file(indir=sshd.home_dir, fname="data-10k", fsize=10*1024)
+        env.make_data_file(indir=sshd.home_dir, fname="data-10m", fsize=10*1024*1024)
+        env.make_data_file(indir=env.gen_dir, fname="data-10k", fsize=10*1024)
+        env.make_data_file(indir=env.gen_dir, fname="data-10m", fsize=10*1024*1024)
+
+    def test_51_01_insecure(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--insecure',
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+
+    def test_51_02_unknown_hosts(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.unknown_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(60)  # CURLE_PEER_FAILED_VERIFICATION
+
+    def test_51_03_known_hosts(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+
+    # use key not in authorized_keys file
+    def test_51_04_unauth_user(self, env: Env, sshd: Sshd):
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user2_pubkey_file,
+            '--key', sshd.user2_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(67)  # CURLE_LOGIN_DENIED
+
+    def test_51_10_dl_single(self, env: Env, sshd: Sshd):
+        count = 1
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_51_11_dl_serial(self, env: Env, sshd: Sshd):
+        count = 5
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.ssh_download(urls=[url], extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_51_12_dl_parallel(self, env: Env, sshd: Sshd):
+        count = 5
+        curl = CurlClient(env=env)
+        doc_file = os.path.join(sshd.home_dir, 'data-10k')
+        url = f'sftp://{env.domain1}:{sshd.port}/{doc_file}?[0-{count-1}]'
+        r = curl.http_download(urls=[url], extra_args=[
+            '--parallel',
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_downloads(curl, doc_file, count)
+
+    def test_51_20_ul_single(self, env: Env, sshd: Sshd):
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_20.data')
+        curl = CurlClient(env=env)
+        url = f'sftp://{env.domain1}:{sshd.port}/{destfile}'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        self.check_upload(sshd, srcfile, destfile)
+
+    def test_51_21_ul_serial(self, env: Env, sshd: Sshd):
+        count = 5
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_21.data')
+        curl = CurlClient(env=env)
+        url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        for i in range(count):
+            self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+    def test_51_22_ul_parallel(self, env: Env, sshd: Sshd):
+        count = 5
+        srcfile = os.path.join(env.gen_dir, 'data-10k')
+        destfile = os.path.join(sshd.home_dir, 'upload_22.data')
+        curl = CurlClient(env=env)
+        url = f'sftp://{env.domain1}:{sshd.port}/{destfile}.[0-{count-1}]'
+        r = curl.ssh_upload(urls=[url], fupload=srcfile, extra_args=[
+            '--parallel',
+            '--knownhosts', sshd.known_hosts,
+            '--pubkey', sshd.user1_pubkey_file,
+            '--key', sshd.user1_privkey_file,
+            '--user', f'{os.environ["USER"]}:',
+        ])
+        r.check_exit_code(0)
+        for i in range(count):
+            self.check_upload(sshd, srcfile, f'{destfile}.{i}')
+
+    def check_downloads(self, client, srcfile: str, count: int,
+                        complete: bool = True):
+        for i in range(count):
+            dfile = client.download_file(i)
+            assert os.path.exists(dfile)
+            if complete and not filecmp.cmp(srcfile, dfile, shallow=False):
+                diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+                                                    b=open(dfile).readlines(),
+                                                    fromfile=srcfile,
+                                                    tofile=dfile,
+                                                    n=1))
+                assert False, f'download {dfile} differs:\n{diff}'
+
+    def check_upload(self, sshd: Sshd, srcfile, destfile, binary=True):
+        assert os.path.exists(srcfile)
+        assert os.path.exists(destfile)
+        if not filecmp.cmp(srcfile, destfile, shallow=False):
+            diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(),
+                                                b=open(destfile).readlines(),
+                                                fromfile=srcfile,
+                                                tofile=destfile,
+                                                n=1))
+            assert not binary and len(diff) == 0, f'upload {destfile} differs:\n{diff}'

tests/http/testenv/__init__.py

@@ -37,3 +37,4 @@ from .client import LocalClient
 from .nghttpx import Nghttpx, NghttpxQuic, NghttpxFwd
 from .vsftpd import VsFTPD
 from .dante import Dante
+from .sshd import Sshd

tests/http/testenv/curl.py

@@ -916,6 +916,65 @@ class CurlClient:
                                with_tcpdump=with_tcpdump,
                                extra_args=extra_args)
 
+    def ssh_download(self, urls: List[str],
+                     with_stats: bool = True,
+                     with_profile: bool = False,
+                     with_tcpdump: bool = False,
+                     no_save: bool = False,
+                     extra_args: Optional[List[str]] = None):
+        if extra_args is None:
+            extra_args = []
+        if no_save:
+            extra_args.extend([
+                '--out-null',
+            ])
+        else:
+            extra_args.extend([
+                '-o', 'download_#1.data',
+            ])
+        # remove any existing ones
+        for i in range(100):
+            self._rmf(self.download_file(i))
+        if with_stats:
+            extra_args.extend([
+                '-w', '%{json}\\n'
+            ])
+        return self._raw(urls, options=extra_args,
+                         with_stats=with_stats,
+                         with_headers=False,
+                         with_profile=with_profile,
+                         with_tcpdump=with_tcpdump)
+
+    def ssh_upload(self, urls: List[str],
+                   fupload: Optional[Any] = None,
+                   updata: Optional[str] = None,
+                   with_stats: bool = True,
+                   with_profile: bool = False,
+                   with_tcpdump: bool = False,
+                   extra_args: Optional[List[str]] = None):
+        if extra_args is None:
+            extra_args = []
+        if fupload is not None:
+            extra_args.extend([
+                '--upload-file', fupload
+            ])
+        elif updata is not None:
+            extra_args.extend([
+                '--upload-file', '-'
+            ])
+        else:
+            raise Exception('need either file or data to upload')
+        if with_stats:
+            extra_args.extend([
+                '-w', '%{json}\\n'
+            ])
+        return self._raw(urls, options=extra_args,
+                         intext=updata,
+                         with_stats=with_stats,
+                         with_headers=False,
+                         with_profile=with_profile,
+                         with_tcpdump=with_tcpdump)
+
     def response_file(self, idx: int):
         return os.path.join(self._run_dir, f'download_{idx}.data')
 

tests/http/testenv/env.py

@@ -298,6 +298,35 @@ class EnvConfig:
             except Exception:
                 self.danted = None
 
+        self.sshd = self.config['sshd']['sshd']
+        if self.sshd == '':
+            self.sshd = None
+        self._sshd_version = None
+        if self.sshd is not None:
+            try:
+                p = subprocess.run(args=[self.sshd, '-V'],
+                                   capture_output=True, text=True)
+                assert p.returncode == 0
+                if p.returncode != 0:
+                    self.sshd = None
+                else:
+                    m = re.match(r'^OpenSSH_(\d+\.\d+.*),.*', p.stderr)
+                    assert m, f'version: {p.stderr}'
+                    if m:
+                        self._sshd_version = m.group(1)
+                    else:
+                        self.sshd = None
+                        raise Exception(f'Unable to determine sshd version from: {p.stderr}')
+            except Exception:
+                self.sshd = None
+
+        if self.sshd:
+            self.sftpd = self.config['sshd']['sftpd']
+            if self.sftpd == '':
+                self.sftpd = None
+        else:
+            self.sftpd = None
+
         self._tcpdump = shutil.which('tcpdump')
 
     @property
@@ -576,6 +605,14 @@ class Env:
     def has_danted() -> bool:
         return Env.CONFIG.danted is not None
 
+    @staticmethod
+    def has_sshd() -> bool:
+        return Env.CONFIG.sshd is not None
+
+    @staticmethod
+    def has_sftpd() -> bool:
+        return Env.has_sshd() and Env.CONFIG.sftpd is not None
+
     @staticmethod
     def tcpdump() -> Optional[str]:
         return Env.CONFIG.tcpdmp

tests/http/testenv/sshd.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+# SPDX-License-Identifier: curl
+#
+###########################################################################
+#
+import logging
+import os
+import socket
+import stat
+import subprocess
+import time
+from datetime import timedelta, datetime
+
+from typing import Dict
+
+from . import CurlClient
+from .env import Env
+from .ports import alloc_ports_and_do
+
+log = logging.getLogger(__name__)
+
+
+class Sshd:
+
+    def __init__(self, env: Env):
+        self.env = env
+        self._cmd = Env.CONFIG.sshd
+        self._sftpd = Env.CONFIG.sftpd
+        self._keygen = 'ssh-keygen'
+        self._port = 0
+        self.name = 'sshd'
+        self._port_skey = 'sshd'
+        self._port_specs = {
+            'sshd': socket.SOCK_STREAM,
+        }
+        self._sshd_dir = os.path.join(env.gen_dir, self.name)
+        self._home_dir = os.path.join(self._sshd_dir, 'home')
+        self._run_dir = os.path.join(self._sshd_dir, 'run')
+        self._tmp_dir = os.path.join(self._sshd_dir, 'tmp')
+        self._conf_file = os.path.join(self._sshd_dir, 'test.conf')
+        self._auth_keys = os.path.join(self._sshd_dir, 'authorized_keys')
+        self._known_hosts = os.path.join(self._sshd_dir, 'known_hosts')
+        self._unknown_hosts = os.path.join(self._sshd_dir, 'unknown_hosts')
+        self._sshd_log = os.path.join(self._sshd_dir, 'sshd.log')
+        self._pid_file = os.path.join(self._sshd_dir, 'sshd.pid')
+        self._key_algs = [
+            'rsa', 'ecdsa', 'ed25519',
+        ]
+        self._host_key_files = []
+        self._host_pub_files = []
+        self._users = [
+            'user1',
+            'user2',
+        ]
+        self._user_key_files = []
+        self._user_pub_files = []
+        self._process = None
+
+        self.clear_logs()
+        self._mkpath(self._home_dir)
+        env.make_data_file(indir=self._home_dir, fname="data", fsize=1024)
+        self._mkpath(self._tmp_dir)
+
+    @property
+    def port(self) -> int:
+        return self._port
+
+    @property
+    def home_dir(self):
+        return self._home_dir
+
+    @property
+    def known_hosts(self):
+        return self._known_hosts
+
+    @property
+    def unknown_hosts(self):
+        return self._unknown_hosts
+
+    @property
+    def user1_pubkey_file(self):
+        return self._user_pub_files[0]
+
+    @property
+    def user1_privkey_file(self):
+        return self._user_key_files[0]
+
+    @property
+    def user2_pubkey_file(self):
+        return self._user_pub_files[1]
+
+    @property
+    def user2_privkey_file(self):
+        return self._user_key_files[1]
+
+    def mk_host_keys(self):
+        self._host_key_files = []
+        self._host_pub_files = []
+        # a known_host file that knows all our test host pubkeys
+        with open(self._unknown_hosts, 'w') as fd_unknown, \
+             open(self._known_hosts, 'w') as fd_known:
+            os.chmod(self._unknown_hosts, stat.S_IRUSR | stat.S_IWUSR)
+            os.chmod(self._known_hosts, stat.S_IRUSR | stat.S_IWUSR)
+            for alg in self._key_algs:
+                key_file = os.path.join(self._sshd_dir, f'ssh_host_{alg}_key')
+                if not os.path.exists(key_file):
+                    p = subprocess.run(args=[
+                        self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
+                    ], capture_output=True, text=True)
+                    if p.returncode != 0:
+                        raise RuntimeError(f'error generating host key {key_file}: {p.returncode}')
+                self._host_key_files.append(key_file)
+                pub_file = f'{key_file}.pub'
+                self._host_pub_files.append(pub_file)
+                pubkey = open(pub_file).read()
+                # fd_known.write(f'[127.0.0.1]:{self.port} {pubkey}')
+                fd_known.write(f'[{self.env.domain1.lower()}]:{self.port} {pubkey}')
+                fd_unknown.write(f'dummy.invalid {pubkey}')
+        # hash the known_hosts file, libssh requires it
+        p = subprocess.run(args=[
+            self._keygen, '-H', '-f', self._known_hosts
+        ], capture_output=True, text=True)
+        if p.returncode != 0:
+            raise RuntimeError(f'error hashing {self._known_hosts}: {p.returncode}')
+
+    def mk_user_keys(self):
+        self._user_key_files = []
+        self._user_pub_files = []
+        alg = 'rsa'
+        for user in self._users:
+            key_file = os.path.join(self._sshd_dir, f'id_{user}_user_{alg}_key')
+            if not os.path.exists(key_file):
+                p = subprocess.run(args=[
+                    self._keygen, '-q', '-N', '', '-t', alg, '-f', key_file
+                ], capture_output=True, text=True)
+                if p.returncode != 0:
+                    raise RuntimeError(f'error generating user key {key_file}: {p.returncode}')
+            self._user_key_files.append(key_file)
+            self._user_pub_files.append(f'{key_file}.pub')
+        with open(self._auth_keys, 'w') as fd:
+            os.chmod(self._auth_keys, stat.S_IRUSR | stat.S_IWUSR)
+            pubkey = open(self._user_pub_files[0]).read()
+            fd.write(pubkey)
+
+    def clear_logs(self):
+        self._rmf(self._sshd_log)
+
+    def dump_log(self):
+        lines = ['>>--sshd log ----------------------------------------------\n']
+        lines.extend(open(self._sshd_log))
+        lines.extend(['>>--curl log ----------------------------------------------\n'])
+        lines.extend(open(os.path.join(self._tmp_dir, 'curl.stderr')))
+        lines.append('<<-------------------------------------------------------\n')
+        return ''.join(lines)
+
+    def exists(self):
+        return os.path.exists(self._cmd)
+
+    def is_running(self):
+        if self._process:
+            self._process.poll()
+            return self._process.returncode is None
+        return False
+
+    def start_if_needed(self):
+        if not self.is_running():
+            return self.start()
+        return True
+
+    def stop(self, wait_dead=True):
+        self._mkpath(self._tmp_dir)
+        if self._process:
+            self._process.terminate()
+            self._process.wait(timeout=2)
+            self._process = None
+            return not wait_dead or True
+        return True
+
+    def restart(self):
+        self.stop()
+        return self.start()
+
+    def initial_start(self):
+
+        def startup(ports: Dict[str, int]) -> bool:
+            self._port = ports[self._port_skey]
+            self.mk_host_keys()
+            self.mk_user_keys()
+            if self.start():
+                self.env.update_ports(ports)
+                return True
+            self.stop()
+            self._port = 0
+            return False
+
+        return alloc_ports_and_do(self._port_specs, startup,
+                                  self.env.gen_root, max_tries=3)
+
+    def start(self, wait_live=True):
+        assert self._port > 0
+        self._mkpath(self._tmp_dir)
+        if self._process:
+            self.stop()
+        self._write_config()
+        args = [
+            self._cmd,
+            '-D',
+            '-f', f'{self._conf_file}',
+            '-E', f'{self._sshd_log}',
+        ]
+        run_env = os.environ.copy()
+        # does not have any effect, sadly
+        # run_env['HOME'] = f'{self._home_dir}'
+        procerr = open(self._sshd_log, 'a')
+        self._process = subprocess.Popen(args=args, stderr=procerr, env=run_env)
+        if self._process.returncode is not None:
+            return False
+        return self.wait_live(timeout=timedelta(seconds=Env.SERVER_TIMEOUT))
+
+    def wait_live(self, timeout: timedelta):
+        curl = CurlClient(env=self.env, run_dir=self._tmp_dir,
+                          timeout=timeout.total_seconds())
+        try_until = datetime.now() + timeout
+        while datetime.now() < try_until:
+            r = curl.http_get(url=f'scp://{self.env.domain1}:{self._port}/{self.home_dir}/data',
+                              extra_args=[
+                                  '--insecure',
+                                  '--pubkey', self.user1_pubkey_file,
+                                  '--key', self.user1_privkey_file,
+                                  '--user', f'{os.environ["USER"]}:',
+                              ])
+            if r.exit_code == 0:
+                return True
+            time.sleep(.1)
+        log.error(f"sshd still not responding after {timeout}")
+        return False
+
+    def _rmf(self, path):
+        if os.path.exists(path):
+            return os.remove(path)
+
+    def _mkpath(self, path):
+        if not os.path.exists(path):
+            return os.makedirs(path)
+
+    def _write_config(self):
+        conf = [
+            f'ListenAddress 127.0.0.1:{self._port}',
+            'AllowTcpForwarding yes',
+            'AuthenticationMethods publickey',
+            'PasswordAuthentication no',
+            # in CI, we might run as root, allow this
+            'PermitRootLogin yes',
+            'LogLevel VERBOSE',
+            f'AuthorizedKeysFile {self._auth_keys}',
+            f'PidFile {self._pid_file}',
+        ]
+        conf.extend([f'HostKey {key_file}' for key_file in self._host_key_files])
+        if self._sftpd:
+            conf.append(f'Subsystem sftp {self._sftpd}')
+        conf.append('\n')
+        with open(self._conf_file, 'w') as fd:
+            fd.write("\n".join(conf))