From b9e9780dae555a4342577200cb0fb3af3b63634a Mon Sep 17 00:00:00 2001 From: Ian Norton Date: Thu, 19 May 2016 11:31:55 +0100 Subject: [PATCH] speed test harness for cached builds of openssl --- speedtest/fixtures.py | 21 ++++ speedtest/test_performance_curl.py | 107 ++++++++++++++++++ speedtest/test_performance_openssl.py | 149 ++++++++++++++++++++++++++ speedtest/utils.py | 130 ++++++++++++++++++++++ 4 files changed, 407 insertions(+) create mode 100644 speedtest/fixtures.py create mode 100644 speedtest/test_performance_curl.py create mode 100644 speedtest/test_performance_openssl.py create mode 100644 speedtest/utils.py diff --git a/speedtest/fixtures.py b/speedtest/fixtures.py new file mode 100644 index 00000000..e8cb7580 --- /dev/null +++ b/speedtest/fixtures.py @@ -0,0 +1,21 @@ +""" +Test fixtures +""" + +import os +import pytest +from utils import find_visual_studio, get_vc_envs + + +@pytest.fixture() +def clcache_envs(): + """ + return a dict of envs suitable for clcache to work with + :return: + """ + vcdir, _ = find_visual_studio() + envs = get_vc_envs() + cachedir = os.path.join("clcache_cachedir") + envs["CLCACHE_DIR"] = cachedir + envs["CLCACHE_CL"] = os.path.join(vcdir, "cl.exe") + return envs diff --git a/speedtest/test_performance_curl.py b/speedtest/test_performance_curl.py new file mode 100644 index 00000000..a9887c2c --- /dev/null +++ b/speedtest/test_performance_curl.py @@ -0,0 +1,107 @@ +#!/usr/bin/python +""" +A py.test test that attempts to build libcurl and benchmark the effect of clcache +""" + +import sys +import os +import pytest +import urllib +import zipfile +import subprocess + +from utils import block_message, retry_delete, common_module_setup, check_program, get_vc_envs, download_file +from utils import DISTDIR, RETAIN_CACHE, EMPTY_CACHE + +from fixtures import clcache_envs + +CURL_URL = "https://github.com/curl/curl/archive/curl-7_49_0.zip" +CURL_ZIP = "curl-7_49_0.zip" +SOURCES = None + + +def clean_build(): + """ + Unpack the curl source, possibly deleting the previous one + :return: + """ + with zipfile.ZipFile(CURL_ZIP, "r") as unzip: + folder = unzip.namelist()[0] + if os.path.exists(folder): + with block_message("delete old curl folder"): + retry_delete(folder) + + with block_message("unzip curl"): + unzip.extractall() + + global SOURCES + SOURCES = folder.rstrip("/") + + +def setup_function(request): + """ + Ensure a clean build tree before each test + :return: + """ + os.chdir(DISTDIR) + clean_build() + + +def setup_module(): + """ + Check that our exe has been built. + :return: + """ + common_module_setup() + download_file(CURL_URL, CURL_ZIP) + + +def build_curl(addpath=None, envs=get_vc_envs(), pdbs=False): + """ + Build curl with nmake + :return: + """ + check_program(envs, "nmake.exe") + workdir = os.path.join(SOURCES, "winbuild") + + if addpath is not None: + envs["PATH"] = addpath + os.pathsep + envs["PATH"] + + gen_pdbs = "GEN_PDB=no" + if pdbs: + gen_pdbs = "GEN_PDB=yes" + + with block_message("build curl"): + subprocess.check_output(["nmake", "-f", "Makefile.vc", "mode=static", + "ENABLE_SSPI=no", + "ENABLE_IPV6=no", + "ENABLE_IDN=no", + gen_pdbs], + shell=True, + cwd=workdir, + env=envs) + + +def test_build_nocache(): + """ + Build curl with no caching + :return: + """ + build_curl() + + +@pytest.mark.parametrize("cache_setting", [EMPTY_CACHE, RETAIN_CACHE]) +def test_build_withclcache(clcache_envs, cache_setting): + """ + Time a curl build with a cold cache + :param clcache_envs: clcache environment vars fixture + :param cache_setting: if True, delete the cache from disk + :return: + """ + if cache_setting == EMPTY_CACHE: + retry_delete(clcache_envs["CLCACHE_DIR"]) + build_curl(DISTDIR, clcache_envs) + + +if __name__ == "__main__": + pytest.main(sys.argv[1:]) diff --git a/speedtest/test_performance_openssl.py b/speedtest/test_performance_openssl.py new file mode 100644 index 00000000..af7bd835 --- /dev/null +++ b/speedtest/test_performance_openssl.py @@ -0,0 +1,149 @@ +#!/usr/bin/python +""" +A py.test test that attempts to build openssl and benchmark the effect of clcache +""" +import sys +import os +import pytest +import zipfile +import subprocess + +from utils import block_message, retry_delete, common_module_setup, check_program, get_vc_envs, download_file +from utils import DISTDIR, RETAIN_CACHE, EMPTY_CACHE + +from fixtures import clcache_envs + +OPENSSL_ZIP = "OpenSSL_1_0_2-stable.zip" +OPENSSL_URL = "https://codeload.github.com/openssl/openssl/zip/OpenSSL_1_0_2-stable" +SOURCES = None + + +def clean_build(): + """ + Unpack the openssl source, possibly deleting the previous one + :return: + """ + with zipfile.ZipFile(OPENSSL_ZIP, "r") as unzip: + folder = unzip.namelist()[0] + if os.path.exists(folder): + with block_message("delete old openssl folder"): + retry_delete(folder) + + with block_message("unzip openssl"): + unzip.extractall() + + global SOURCES + SOURCES = folder.rstrip("/") + + +def configure_openssl(envs): + """ + Run the configure steps (requires perl) + :param envs: + :return: + """ + check_program(envs, "nmake.exe") + check_program(envs, "perl.exe") + + with block_message("configure openssl"): + subprocess.check_call(["perl", + "Configure", "VC-WIN32", "no-asm", "--prefix=c:\openssl"], + env=envs, + cwd=SOURCES) + + with block_message("generate makefiles"): + subprocess.check_call([os.path.join("ms", "do_ms.bat")], + shell=True, + env=envs, + cwd=SOURCES) + + +def setup_function(request): + """ + Ensure a clean build tree before each test + :return: + """ + os.chdir(DISTDIR) + clean_build() + configure_openssl(get_vc_envs()) + + +def setup_module(): + """ + Check that our exe has been built. + :return: + """ + common_module_setup() + download_file(OPENSSL_URL, OPENSSL_ZIP) + + +def replace_wipe_cflags(filename): + """ + Open the nmake file given and turn off PDB generation for .obj files + :param filename: + :return: + """ + lines = [] + with open(filename, "rb") as makefile: + for line in makefile.readlines(): + if line.startswith("APP_CFLAG="): + lines.append("APP_CFLAG=") + elif line.startswith("LIB_CFLAG="): + lines.append("LIB_CFLAG=/Zl") + else: + lines.append(line.rstrip()) + + with open(filename, "wb") as makefile: + for line in lines: + makefile.write(line + "\r\n") + + +def build_openssl(addpath=None, envs=get_vc_envs(), pdbs=False): + """ + Build openssl, optionally prefixing addpath to $PATH + :param addpath: + :param envs: env var dict to use + :param pdbs: if False, turn off pdb generation in the makefile + :return: + """ + nmakefile = os.path.join("ms", "nt.mak") + if not pdbs: + replace_wipe_cflags(os.path.join(SOURCES, nmakefile)) + + if addpath is not None: + envs["PATH"] = addpath + os.pathsep + envs["PATH"] + + try: + with block_message("running nmake"): + subprocess.check_output(["nmake", "-f", nmakefile], + shell=True, + env=envs, + cwd=SOURCES) + except subprocess.CalledProcessError as cpe: + print cpe.output + raise + + +def test_build_nocache(): + """ + Time an openssl build with no caching involved at all + :return: + """ + build_openssl() + + +@pytest.mark.parametrize("cache_setting", [EMPTY_CACHE, RETAIN_CACHE]) +def test_build_withclcache(clcache_envs, cache_setting): + """ + Time an openssl build with a cold cache + :param clcache_envs: clcache environment vars fixture + :param cache_setting: if True, delete the cache from disk + :return: + """ + if cache_setting == EMPTY_CACHE: + retry_delete(clcache_envs["CLCACHE_DIR"]) + build_openssl(DISTDIR, clcache_envs) + + +if __name__ == "__main__": + pytest.main(sys.argv[1:]) diff --git a/speedtest/utils.py b/speedtest/utils.py new file mode 100644 index 00000000..bb0e607b --- /dev/null +++ b/speedtest/utils.py @@ -0,0 +1,130 @@ +""" +Common test functions +""" +import os +import urllib +import shutil +import pytest +import time +import subprocess + +from contextlib import contextmanager + +THISDIR = os.path.dirname(os.path.abspath(__file__)) +DISTDIR = os.path.join(os.path.dirname(THISDIR), "dist") +CLCACHE = os.path.join(DISTDIR, "cl.exe") + +EMPTY_CACHE = "empty_cache" +RETAIN_CACHE = "retain_cache" + + +def check_program(envs, program): + """ + Skip the current test/fixture if program is not in envs[PATH] + :param envs: look for PATH in this env dict + :param program: program (eg, perl.exe) to look for + :return: + """ + assert envs is not None + assert "PATH" in envs + + for pathdir in envs["PATH"].split(os.pathsep): + if os.path.isfile(os.path.join(pathdir, program)): + return True + pytest.skip("cannot find {} on PATH".format(program)) + + +def retry_delete(path): + """ + Repeatedly attempt to delete path + :param path: + :return: + """ + for _ in range(30): + # antivirus might be busy in here.. + try: + shutil.rmtree(path) + return + except WindowsError: + time.sleep(1) + if os.path.exists(path): + raise Exception("could not delete {}".format(path)) + + +@contextmanager +def block_message(message): + """ + Emit "begin .. end" messages for a block of code + :param message: + """ + started = time.time() + print "\n..begin {} .. ".format(message) + + try: + yield + result = "OK" + except: + result = "ERROR" + raise + finally: + print "\n..end {} {}.. ({}sec)".format(message, result, time.time() - started) + + +def find_visual_studio(): + """ + Attempt to find vs 11, 12 or 13 + :return: + """ + vcvers = ["13.0", "12.0", "11.0"] + for vc in vcvers: + vcdir = os.path.join("c:\\", "Program Files (x86)", + "Microsoft Visual Studio {}".format(vc), + "VC", "bin") + vcvars = os.path.join(vcdir, "vcvars32.bat") + if os.path.exists(vcvars): + return vcdir, vcvars + + raise Exception("cannot find visual studio!") + + +def get_vc_envs(): + """ + Get the visual studio dev env vars + :return: + """ + if get_vc_envs.envs is None: + envs = dict(os.environ) + _, vcvars = find_visual_studio() + with block_message("getting vc envs"): + getenvs = subprocess.check_output([vcvars, '>', 'NUL', '&&', 'set'], shell=True) + for line in getenvs.splitlines(): + if "=" in line: + name, val = line.split("=", 1) + envs[name.upper()] = val + get_vc_envs.envs = envs + return get_vc_envs.envs +get_vc_envs.envs = None + + +def download_file(url, localfile): + """ + Download the given url and save it at localfile + :param url: + :param localfile: + :return: + """ + if not os.path.exists(localfile): + with block_message("download " + localfile): + urllib.urlretrieve(url, localfile + ".part") + os.rename(localfile + ".part", localfile) + + +def common_module_setup(): + """ + Called by setup_module() in tests + :return: + """ + os.chdir(DISTDIR) + if not os.path.isfile(CLCACHE): + pytest.fail("please build the exe first") + find_visual_studio()