diff --git a/benchmarks/Neo.Benchmarks/NATIVE-CONTRACT-BENCHMARKS.md b/benchmarks/Neo.Benchmarks/NATIVE-CONTRACT-BENCHMARKS.md new file mode 100644 index 0000000000..b93828fed9 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NATIVE-CONTRACT-BENCHMARKS.md @@ -0,0 +1,98 @@ +Native Contract Benchmarks +========================== + +This suite exercises every discoverable native contract method in the Neo +runtime, fabricates the chain state those contracts expect (committee witnesses, +oracle requests, notary deposits, etc.), and reports their execution cost across +the canonical input profiles (`Tiny`, `Small`, `Medium`, `Large`). + +Running the suite +----------------- + +There are two runners: + +### Manual harness (default & CI-friendly) + +``` +dotnet run -c Release --framework net10.0 -- \ + --native-manual-run \ + --native-iterations 3 \ + --native-warmup 0 \ + --native-output BenchmarkDotNet.Artifacts/manual +``` + +The manual runner: + +- Discovers contracts/methods via reflection and generates benchmark cases for + every profile. +- Spins up a seeded `NeoSystem` so witness checks, notary validation, and + candidate registration succeed without special-casing individual contracts. +- Logs progress to stdout (add `--native-verbose` or + `NEO_NATIVE_BENCH_VERBOSE=1` for per-case telemetry). +- Emits `BenchmarkDotNet.Artifacts/manual/manual-native-contract-summary.txt` + plus a JSON companion file. Both include coverage counts, per-method stats, + and a list of any filtered cases. +- Supports inline filter overrides. Pass `--native-contract`, `--native-method`, + `--native-sizes`, `--native-limit`, or `--native-job` to override the + corresponding environment variables for a single run (values can be comma or + space separated, just like the env-based filters). + +### Shortcut script + +For convenience, the repository ships with +`scripts/run-native-benchmarks.sh`, which wraps the manual runner and exposes +the most common options: + +``` +scripts/run-native-benchmarks.sh \ + --contract NeoToken \ + --method "get*,onNEP17Payment" \ + --sizes Tiny,Small \ + --limit 50 \ + --iterations 3 \ + --warmup 0 \ + --verbose +``` + +Any additional arguments placed after `--` are forwarded directly to `dotnet +run`, so you can still customise the build configuration or framework. + +By default the manual runner uses the *Balanced* profile (20 measured iterations, +3 warmup passes, 10% trimmed mean). Switch to `--native-job quick` for the old +smoke-test behaviour, or `--native-job thorough` for 40 iterations / 5 warmups +with heavier outlier trimming. + +### BenchmarkDotNet (high-fidelity measurements) + +``` +dotnet run -c Release --framework net10.0 -- -f '*NativeContractMethodBenchmarks*' +``` + +Use this variant when you can afford a longer run and want BDN's job/diagnostic +output. It relies on the same discovery and argument generation pipeline but +lets you pick the built-in BDN jobs via `NEO_NATIVE_BENCH_JOB=Quick|Short|Default`. + +Focusing on a subset +-------------------- + +Running every contract * every input size takes time. Narrow a run with the +following environment variables (applies to both runners unless noted): + +| Variable | Description | +|----------|-------------| +| `NEO_NATIVE_BENCH_CONTRACT` | Comma/space-separated wildcard patterns that match contract names (e.g. `StdLib, NeoToken`). | +| `NEO_NATIVE_BENCH_METHOD` | Wildcard patterns applied to method names (case-insensitive, e.g. `get*`, `*Payment`). | +| `NEO_NATIVE_BENCH_SIZES` | Restrict workload sizes (`Tiny`, `Small`, `Medium`, `Large`). Multiple values allowed. | +| `NEO_NATIVE_BENCH_LIMIT` | Stop after the first _N_ benchmark cases. | +| `NEO_NATIVE_BENCH_JOB` | BenchmarkDotNet only - select `Quick`, `Short`, or `Default`. | +| `NEO_NATIVE_BENCH_ITERATIONS` | Manual runner only - override measured iterations per case (default `5`). | +| `NEO_NATIVE_BENCH_WARMUP` | Manual runner only - warmup passes before measuring (default `1`). | +| `NEO_NATIVE_BENCH_VERBOSE` | Manual runner only - `1/true` streams per-case statistics. | + +Example - only benchmark `StdLib` methods with tiny inputs: + +``` +NEO_NATIVE_BENCH_CONTRACT=StdLib \ +NEO_NATIVE_BENCH_SIZES=Tiny \ +dotnet run -c Release --framework net10.0 -- --native-manual-run +``` diff --git a/benchmarks/Neo.Benchmarks/NativeContractMethodBenchmarks.cs b/benchmarks/Neo.Benchmarks/NativeContractMethodBenchmarks.cs new file mode 100644 index 0000000000..a199b1226b --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContractMethodBenchmarks.cs @@ -0,0 +1,60 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractMethodBenchmarks.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using System; +using System.Collections.Generic; + +namespace Neo.Benchmarks.NativeContracts +{ + [MemoryDiagnoser] + public class NativeContractMethodBenchmarks : IConfigSource, IDisposable + { + private readonly NativeContractBenchmarkSuite _suite; + private NativeContractBenchmarkInvoker _invoker; + + public NativeContractMethodBenchmarks() + { + _suite = NativeContractBenchmarkSuite.CreateDefault(); + Config = new NativeContractBenchmarkConfig(_suite); + } + + public IConfig Config { get; } + + [ParamsSource(nameof(GetCases))] + public NativeContractBenchmarkCase Case { get; set; } + + public IEnumerable GetCases() => _suite.Cases; + + [GlobalSetup] + public void GlobalSetup() + { + if (Case is null) + throw new InvalidOperationException("Benchmark case not set. Ensure discovery produced at least one scenario."); + _invoker = _suite.CreateInvoker(Case); + } + + [Benchmark(Description = "Invoke native contract method")] + public object Execute() => _invoker.Invoke(); + + [GlobalCleanup] + public void GlobalCleanup() + { + _invoker = null; + } + + public void Dispose() + { + _suite.Dispose(); + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractArgumentGenerator.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractArgumentGenerator.cs new file mode 100644 index 0000000000..693b68519d --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractArgumentGenerator.cs @@ -0,0 +1,1162 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractArgumentGenerator.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.Cryptography; +using Neo.Cryptography.BLS12_381; +using Neo.Cryptography.ECC; +using Neo.Extensions; +using Neo.Json; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Manifest; +using Neo.SmartContract.Native; +using Neo.VM; +using Neo.VM.Types; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Numerics; +using System.Reflection; +using System.Text; +using VMArray = Neo.VM.Types.Array; +using VMBuffer = Neo.VM.Types.Buffer; +using VMStruct = Neo.VM.Types.Struct; + +#nullable enable + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Builds deterministic argument factories for native contract benchmark scenarios. + /// + public sealed class NativeContractArgumentGenerator + { + private readonly struct KnownFactory + { + public KnownFactory( + Func valueFactory, + Func summaryFactory) + { + ValueFactory = valueFactory; + SummaryFactory = summaryFactory; + } + + public Func ValueFactory { get; } + public Func SummaryFactory { get; } + } + + private readonly ImmutableDictionary _knownFactories; + private static readonly byte[] s_blsScalarOne = Scalar.One.ToArray(); + private static readonly byte[] s_ecdsaMessage = Encoding.ASCII.GetBytes("HelloWorld"); + private static readonly (byte[] PubKey, byte[] Signature) s_secp256k1Sha256Vector = CreateEcdsaVector( + "0B5FB3A050385196B327BE7D86CBCE6E40A04C8832445AF83AD19C82103B3ED9", + "04B6363B353C3EE1620C5AF58594458AA00ABF43A6D134D7C4CB2D901DC0F474FD74C94740BD7169AA0B1EF7BC657E824B1D7F4283C547E7EC18C8576ACF84418A", + ECCurve.Secp256k1, + HashAlgorithm.SHA256); + private static readonly (byte[] PubKey, byte[] Signature) s_secp256r1Sha256Vector = CreateEcdsaVector( + "6E63FDA41E9E3ABA9BB5696D58A75731F044A9BDC48FE546DA571543B2FA460E", + "04CAE768E1CF58D50260CAB808DA8D6D83D5D3AB91EAC41CDCE577CE5862D736413643BDECD6D21C3B66F122AB080F9219204B10AA8BBCEB86C1896974768648F3", + ECCurve.Secp256r1, + HashAlgorithm.SHA256); + + public NativeContractArgumentGenerator() + { + _knownFactories = BuildKnownFactories(); + } + + public bool TryBuildArgumentFactory( + NativeContract contract, + MethodInfo handler, + IReadOnlyList parameters, + NativeContractInputProfile profile, + out Func factory, + out string summary, + out string? failureReason) + { + if (parameters.Count == 0) + { + factory = _ => global::System.Array.Empty(); + summary = "No parameters"; + failureReason = null; + return true; + } + + var generators = new List>(parameters.Count); + var descriptorParts = new List(parameters.Count); + var overrides = new Func?[parameters.Count]; + var overrideDescriptors = new string[parameters.Count]; + + ApplyMethodOverrides(contract, handler, profile, overrides, overrideDescriptors); + + for (int i = 0; i < parameters.Count; i++) + { + var parameter = parameters[i]; + if (overrides[i] is null && parameter.Type == typeof(UInt160)) + { + var slot = i; + overrides[i] = ctx => ctx.GetAccount(profile.Size, slot); + overrideDescriptors[i] = $"UInt160(BenchmarkAccount[{slot}])"; + } + + if (overrides[i] is not null) + { + generators.Add(overrides[i]!); + descriptorParts.Add($"{parameter.Name}:{overrideDescriptors[i]}"); + continue; + } + + if (!TryBuildGenerator(parameter.Type, profile, out var generator, out var descriptor, out failureReason)) + { + factory = null!; + summary = string.Empty; + return false; + } + + generators.Add(generator); + descriptorParts.Add($"{parameter.Name}:{descriptor}"); + } + + summary = string.Join("; ", descriptorParts); + factory = context => + { + var values = new object[generators.Count]; + for (int i = 0; i < generators.Count; i++) + values[i] = generators[i](context); + return values; + }; + failureReason = null; + return true; + } + + private void ApplyMethodOverrides( + NativeContract contract, + MethodInfo handler, + NativeContractInputProfile profile, + Func?[] overrides, + string[] overrideDescriptors) + { + if (contract.Name is nameof(CryptoLib)) + { + switch (handler.Name) + { + case nameof(CryptoLib.Bls12381Serialize): + overrides[0] = _ => new InteropInterface(G1Affine.Generator); + overrideDescriptors[0] = "Interop(G1Affine.Generator)"; + break; + + case nameof(CryptoLib.Bls12381Deserialize): + var bytes = CreateBlsSerializedBytes(profile); + overrides[0] = _ => bytes; + overrideDescriptors[0] = $"byte[{bytes.Length}]"; + break; + + case nameof(CryptoLib.Bls12381Equal): + overrides[0] = _ => new InteropInterface(G1Affine.Generator); + overrideDescriptors[0] = "Interop(G1Affine)"; + overrides[1] = _ => new InteropInterface(G1Affine.Generator); + overrideDescriptors[1] = "Interop(G1Affine)"; + break; + + case nameof(CryptoLib.Bls12381Add): + overrides[0] = _ => new InteropInterface(new G1Projective(G1Affine.Generator)); + overrideDescriptors[0] = "Interop(G1Projective)"; + overrides[1] = _ => new InteropInterface(G1Affine.Identity); + overrideDescriptors[1] = "Interop(G1Affine.Identity)"; + break; + + case nameof(CryptoLib.Bls12381Mul): + overrides[0] = _ => new InteropInterface(Gt.Generator); + overrideDescriptors[0] = "Interop(Gt.Generator)"; + overrides[1] = _ => s_blsScalarOne; + overrideDescriptors[1] = "Scalar(32 bytes)"; + break; + + case nameof(CryptoLib.Bls12381Pairing): + overrides[0] = _ => new InteropInterface(G1Affine.Generator); + overrideDescriptors[0] = "Interop(G1Affine)"; + overrides[1] = _ => new InteropInterface(G2Affine.Generator); + overrideDescriptors[1] = "Interop(G2Affine)"; + break; + + case nameof(CryptoLib.VerifyWithECDsa): + case "VerifyWithECDsaV0": + var ecdsaVector = profile.Size switch + { + NativeContractInputSize.Tiny or NativeContractInputSize.Small => s_secp256k1Sha256Vector, + _ => s_secp256r1Sha256Vector + }; + overrides[0] = _ => s_ecdsaMessage; + overrideDescriptors[0] = "\"HelloWorld\""; + overrides[1] = _ => ecdsaVector.PubKey; + overrideDescriptors[1] = "pubkey(uncompressed)"; + overrides[2] = _ => ecdsaVector.Signature; + overrideDescriptors[2] = "signature"; + break; + } + } + + if (contract.Name is nameof(StdLib)) + { + switch (handler.Name) + { + case nameof(StdLib.Atoi): + overrides[0] = _ => CreateDecimalString(profile); + overrideDescriptors[0] = "numeric string"; + if (overrides.Length > 1) + { + overrides[1] = _ => 10; + overrideDescriptors[1] = "base10"; + } + break; + + case nameof(StdLib.Itoa): + if (overrides.Length > 1) + { + overrides[1] = _ => 10; + overrideDescriptors[1] = "base10"; + } + break; + + case nameof(StdLib.Base64Decode): + overrides[0] = _ => CreateBase64String(profile); + overrideDescriptors[0] = "base64"; + break; + + case nameof(StdLib.Base64UrlDecode): + overrides[0] = _ => CreateBase64UrlString(profile); + overrideDescriptors[0] = "base64url"; + break; + + case nameof(StdLib.Base58Decode): + overrides[0] = _ => CreateBase58String(profile); + overrideDescriptors[0] = "base58"; + break; + + case nameof(StdLib.Base58CheckDecode): + overrides[0] = _ => CreateBase58CheckString(profile); + overrideDescriptors[0] = "base58check"; + break; + + case "Deserialize": + overrides[0] = _ => CreateSerializedStackItemBytes(profile); + overrideDescriptors[0] = "serialized stack item bytes"; + break; + + case "JsonSerialize": + overrides[0] = _ => CreateJsonFriendlyStackItem(profile); + overrideDescriptors[0] = "StackItem(JSON-friendly)"; + break; + + case "JsonDeserialize": + overrides[0] = _ => CreateJsonBytes(profile); + overrideDescriptors[0] = "json bytes"; + break; + + case "MemorySearch": + overrides[0] = _ => Encoding.UTF8.GetBytes("abc"); + overrideDescriptors[0] = "\"abc\" bytes"; + overrides[1] = _ => Encoding.UTF8.GetBytes("c"); + overrideDescriptors[1] = "\"c\" bytes"; + if (overrides.Length > 2) + { + overrides[2] = _ => 0; + overrideDescriptors[2] = "start=0"; + } + if (overrides.Length > 3) + { + overrides[3] = _ => false; + overrideDescriptors[3] = "backward=false"; + } + break; + } + } + + if (contract.Name is nameof(RoleManagement)) + { + switch (handler.Name) + { + case "DesignateAsRole": + var role = ResolveRole(profile.Size); + overrides[0] = _ => role; + overrideDescriptors[0] = $"Role.{role}"; + var nodeCount = DescribeRoleNodeCount(profile); + overrides[1] = ctx => CreateRoleNodes(ctx, profile); + overrideDescriptors[1] = $"ECPoint[{nodeCount}]"; + break; + + case "GetDesignatedByRole": + var queryRole = ResolveRole(profile.Size); + overrides[0] = _ => queryRole; + overrideDescriptors[0] = $"Role.{queryRole}"; + overrides[1] = _ => 0u; + overrideDescriptors[1] = "index=0"; + break; + } + } + + if (contract.Name is nameof(PolicyContract)) + { + switch (handler.Name) + { + case "SetMillisecondsPerBlock": + overrides[0] = ctx => CreatePolicyMilliseconds(ctx); + overrideDescriptors[0] = "milliseconds"; + break; + + case "SetAttributeFeeV0": + overrides[0] = _ => (byte)TransactionAttributeType.HighPriority; + overrideDescriptors[0] = "attribute=HighPriority"; + overrides[1] = _ => CreatePolicyAttributeFee(profile); + overrideDescriptors[1] = "fee"; + break; + + case "SetAttributeFeeV1": + overrides[0] = _ => (byte)TransactionAttributeType.NotaryAssisted; + overrideDescriptors[0] = "attribute=NotaryAssisted"; + overrides[1] = _ => CreatePolicyAttributeFee(profile); + overrideDescriptors[1] = "fee"; + break; + + case "GetAttributeFeeV0": + overrides[0] = _ => (byte)TransactionAttributeType.HighPriority; + overrideDescriptors[0] = "attribute=HighPriority"; + break; + + case "GetAttributeFeeV1": + overrides[0] = _ => (byte)TransactionAttributeType.NotaryAssisted; + overrideDescriptors[0] = "attribute=NotaryAssisted"; + break; + + case "SetFeePerByte": + overrides[0] = _ => CreatePolicyFeePerByte(profile); + overrideDescriptors[0] = "feePerByte"; + break; + + case "SetExecFeeFactor": + overrides[0] = _ => CreatePolicyExecFeeFactor(profile); + overrideDescriptors[0] = "execFeeFactor"; + break; + + case "SetStoragePrice": + overrides[0] = _ => CreatePolicyStoragePrice(profile); + overrideDescriptors[0] = "storagePrice"; + break; + + case "SetMaxValidUntilBlockIncrement": + overrides[0] = ctx => CreatePolicyMaxVub(ctx, profile); + overrideDescriptors[0] = "maxVUB"; + break; + + case "SetMaxTraceableBlocks": + overrides[0] = ctx => CreatePolicyMaxTraceableBlocks(ctx, profile); + overrideDescriptors[0] = "maxTraceable"; + break; + + case "BlockAccount": + case "UnblockAccount": + overrides[0] = ctx => ctx.GetAccount(profile.Size); + overrideDescriptors[0] = "UInt160(BenchmarkAccount)"; + break; + } + } + + if (contract.Name is nameof(OracleContract)) + { + switch (handler.Name) + { + case "SetPrice": + overrides[0] = _ => CreateOraclePrice(profile); + overrideDescriptors[0] = "price"; + break; + case "Request": + overrides[0] = _ => CreateOracleUrl(profile); + overrideDescriptors[0] = "url"; + overrides[1] = _ => CreateOracleFilter(profile); + overrideDescriptors[1] = "filter"; + overrides[2] = _ => CreateOracleCallback(profile); + overrideDescriptors[2] = "callback"; + overrides[3] = _ => CreateOracleUserData(profile); + overrideDescriptors[3] = "userData"; + overrides[4] = _ => CreateOracleGasBudget(profile); + overrideDescriptors[4] = "gasForResponse"; + break; + } + } + + if (contract.Name is nameof(NeoToken)) + { + switch (handler.Name) + { + case "OnNEP17Payment": + overrides[0] = ctx => ctx.PrimaryBenchmarkAccount; + overrideDescriptors[0] = "from"; + overrides[1] = ctx => ctx.NeoRegisterPrice; + overrideDescriptors[1] = "amount(register price)"; + overrides[2] = ctx => CreateNeoCandidateData(ctx); + overrideDescriptors[2] = "candidate pubkey"; + break; + case "RegisterCandidate": + overrides[0] = ctx => + { + var committee = ctx.ProtocolSettings.StandbyCommittee; + return committee.Count > 0 ? committee[0] : ECCurve.Secp256r1.G; + }; + overrideDescriptors[0] = "committee pubkey"; + break; + case "UnclaimedGas": + overrides[0] = ctx => ctx.PrimaryBenchmarkAccount; + overrideDescriptors[0] = "benchmark account"; + overrides[1] = ctx => ctx.SeededLedgerHeight + 1; + overrideDescriptors[1] = "end=seededHeight+1"; + break; + } + } + + if (contract.Name is nameof(Notary)) + { + switch (handler.Name) + { + case "BalanceOf": + case "ExpirationOf": + overrides[0] = ctx => ctx.PrimaryBenchmarkAccount; + overrideDescriptors[0] = "benchmark account"; + break; + case "LockDepositUntil": + overrides[0] = ctx => ctx.PrimaryBenchmarkAccount; + overrideDescriptors[0] = "benchmark account"; + overrides[1] = ctx => CreateNotaryLockHeight(ctx); + overrideDescriptors[1] = "lockHeight"; + break; + case "SetMaxNotValidBeforeDelta": + overrides[0] = ctx => CreateNotaryMaxDelta(ctx); + overrideDescriptors[0] = "maxDelta"; + break; + case "OnNEP17Payment": + overrides[0] = ctx => ctx.PrimaryBenchmarkAccount; + overrideDescriptors[0] = "from=sender"; + overrides[1] = _ => NativeContract.GAS.Factor * 20; + overrideDescriptors[1] = "amount(20 GAS)"; + overrides[2] = ctx => CreateNotaryPaymentData(ctx, profile); + overrideDescriptors[2] = "data=[to,till]"; + break; + case "Verify": + overrides[0] = _ => global::System.Array.Empty(); + overrideDescriptors[0] = "signature"; + break; + case "Withdraw": + overrides[0] = ctx => ctx.NotaryDepositAccount; + overrideDescriptors[0] = "deposit owner"; + overrides[1] = ctx => ctx.GetAccount(profile.Size, 1); + overrideDescriptors[1] = "receiver"; + break; + } + } + + if (contract.Name is nameof(LedgerContract)) + { + switch (handler.Name) + { + case nameof(LedgerContract.GetBlock): + overrides[0] = _ => BitConverter.GetBytes(0u); + overrideDescriptors[0] = "blockIndex(0)"; + break; + case "GetTransactionFromBlock": + overrides[0] = _ => BitConverter.GetBytes(0u); + overrideDescriptors[0] = "blockIndex(0)"; + overrides[1] = _ => 0; + overrideDescriptors[1] = "txIndex(0)"; + break; + } + } + + if (contract.Name is nameof(NeoToken)) + { + switch (handler.Name) + { + case "SetGasPerBlock": + overrides[0] = _ => NativeContract.GAS.Factor * 5; + overrideDescriptors[0] = "gasPerBlock(5 GAS)"; + break; + } + } + + if (contract.Name is nameof(ContractManagement)) + { + var parameterInfos = handler.GetParameters(); + var reflectionOffset = Math.Max(0, parameterInfos.Length - overrideDescriptors.Length); + + switch (handler.Name) + { + case "Deploy": + case "Update": + for (int i = 0; i < overrideDescriptors.Length; i++) + { + var infoIndex = i + reflectionOffset; + if (infoIndex < 0 || infoIndex >= parameterInfos.Length) + continue; + + var parameterInfo = parameterInfos[infoIndex]; + var parameterName = parameterInfo.Name; + if (string.Equals(parameterName, "nefFile", StringComparison.OrdinalIgnoreCase)) + { + overrides[i] = _ => NativeContractBenchmarkArtifacts.CreateBenchmarkNefBytes(profile); + overrideDescriptors[i] = "nef bytes"; + } + else if (string.Equals(parameterName, "manifest", StringComparison.OrdinalIgnoreCase)) + { + overrides[i] = _ => NativeContractBenchmarkArtifacts.CreateBenchmarkManifestBytes(profile); + overrideDescriptors[i] = "manifest bytes"; + } + else if (string.Equals(parameterName, "data", StringComparison.OrdinalIgnoreCase) && + parameterInfo.ParameterType == typeof(StackItem)) + { + overrides[i] = _ => StackItem.Null; + overrideDescriptors[i] = "StackItem.Null"; + } + } + break; + } + } + } + + private bool TryBuildGenerator( + Type type, + NativeContractInputProfile profile, + out Func generator, + out string descriptor, + out string? failureReason, + int depth = 0) + { + if (depth > 8) + { + generator = null!; + descriptor = string.Empty; + failureReason = $"Nested parameter depth {depth} exceeds supported limit for type {type.Name}."; + return false; + } + + if (_knownFactories.TryGetValue(type, out var knownFactory)) + { + generator = ctx => knownFactory.ValueFactory(ctx, profile); + descriptor = knownFactory.SummaryFactory(profile); + failureReason = null; + return true; + } + + if (type.IsEnum) + { + var values = Enum.GetValues(type); + var enumValue = values.Length > 0 ? values.GetValue(0) : Activator.CreateInstance(type); + generator = _ => enumValue!; + descriptor = $"{type.Name}.{enumValue}"; + failureReason = null; + return true; + } + + if (type.IsArray && type != typeof(byte[])) + { + var elementType = type.GetElementType()!; + if (!TryBuildGenerator(elementType, profile, out var elementFactory, out var elementDescriptor, out failureReason, depth + 1)) + { + generator = null!; + descriptor = string.Empty; + return false; + } + + var length = Math.Max(1, profile.ElementCount); + generator = ctx => + { + var array = global::System.Array.CreateInstance(elementType, length); + for (int i = 0; i < length; i++) + array.SetValue(elementFactory(ctx), i); + return array; + }; + descriptor = $"{elementDescriptor}[{length}]"; + return true; + } + + if (typeof(IEnumerable).IsAssignableFrom(type)) + { + var list = Enumerable.Repeat(CreateUInt160(0), Math.Max(1, profile.ElementCount)).ToList(); + generator = _ => list; + descriptor = $"UInt160Enumerable({list.Count})"; + failureReason = null; + return true; + } + + if (typeof(IEnumerable).IsAssignableFrom(type)) + { + var list = Enumerable.Repeat(CreateUInt256(1), Math.Max(1, profile.ElementCount)).ToList(); + generator = _ => list; + descriptor = $"UInt256Enumerable({list.Count})"; + failureReason = null; + return true; + } + + if (type == typeof(object)) + { + generator = _ => new ByteString(CreateBytes(Math.Max(1, profile.ByteLength))); + descriptor = "ByteString(object)"; + failureReason = null; + return true; + } + + if (type == typeof(IReadOnlyStore)) + { + generator = ctx => ctx.StoreView; + descriptor = "StoreView"; + failureReason = null; + return true; + } + + if (typeof(DataCache).IsAssignableFrom(type)) + { + generator = ctx => ctx.GetSnapshot(); + descriptor = "DataCache"; + failureReason = null; + return true; + } + + if (type == typeof(Transaction)) + { + var tx = CreateBenchmarkTransaction(profile); + generator = _ => tx; + descriptor = "BenchmarkTransaction"; + failureReason = null; + return true; + } + + generator = null!; + descriptor = string.Empty; + failureReason = $"Unsupported parameter type {type.FullName}."; + return false; + } + + private ImmutableDictionary BuildKnownFactories() + { + var builder = ImmutableDictionary.CreateBuilder(); + + builder[typeof(bool)] = new KnownFactory( + (_, profile) => profile.Size != NativeContractInputSize.Tiny, + profile => $"bool={profile.Size != NativeContractInputSize.Tiny}"); + + builder[typeof(byte)] = new KnownFactory( + (_, profile) => (byte)(profile.ByteLength % byte.MaxValue), + profile => $"byte={profile.ByteLength % byte.MaxValue}"); + + builder[typeof(sbyte)] = new KnownFactory( + (_, profile) => (sbyte)(profile.ByteLength % sbyte.MaxValue), + _ => "sbyte"); + + builder[typeof(short)] = new KnownFactory( + (_, profile) => (short)(profile.ByteLength % short.MaxValue), + _ => "short"); + + builder[typeof(ushort)] = new KnownFactory( + (_, profile) => (ushort)(profile.ByteLength % ushort.MaxValue), + _ => "ushort"); + + builder[typeof(int)] = new KnownFactory( + (_, profile) => ClampToInt(profile.IntegerMagnitude), + profile => $"int={ClampToInt(profile.IntegerMagnitude)}"); + + builder[typeof(uint)] = new KnownFactory( + (_, profile) => ClampToUInt(profile.IntegerMagnitude), + profile => $"uint={ClampToUInt(profile.IntegerMagnitude)}"); + + builder[typeof(long)] = new KnownFactory( + (_, profile) => ClampToLong(profile.IntegerMagnitude), + profile => $"long={ClampToLong(profile.IntegerMagnitude)}"); + + builder[typeof(ulong)] = new KnownFactory( + (_, profile) => ClampToULong(profile.IntegerMagnitude), + profile => $"ulong={ClampToULong(profile.IntegerMagnitude)}"); + + builder[typeof(BigInteger)] = new KnownFactory( + (_, profile) => profile.IntegerMagnitude, + profile => + { + var bits = profile.IntegerMagnitude.IsZero ? 0 : (int)profile.IntegerMagnitude.GetBitLength(); + return $"BigInt[{bits}-bits]"; + }); + + builder[typeof(byte[])] = new KnownFactory( + (_, profile) => CreateBytes(profile.ByteLength), + profile => $"byte[{Math.Max(1, profile.ByteLength)}]"); + + builder[typeof(ReadOnlyMemory)] = new KnownFactory( + (_, profile) => new ReadOnlyMemory(CreateBytes(profile.ByteLength)), + profile => $"ReadOnlyMemory[{Math.Max(1, profile.ByteLength)}]"); + + builder[typeof(string)] = new KnownFactory( + (_, profile) => new string('a', Math.Max(1, profile.ByteLength)), + profile => $"string({Math.Max(1, profile.ByteLength)})"); + + builder[typeof(UInt160)] = new KnownFactory( + (_, profile) => CreateUInt160((int)profile.Size + 1), + profile => $"UInt160({profile.Size})"); + + builder[typeof(UInt256)] = new KnownFactory( + (_, profile) => CreateUInt256((int)profile.Size + 1), + profile => $"UInt256({profile.Size})"); + + builder[typeof(ECPoint)] = new KnownFactory( + (ctx, profile) => + { + var committee = ctx.ProtocolSettings.StandbyCommittee; + if (committee.Count > 0) + { + var index = Math.Min((int)profile.Size, committee.Count - 1); + return committee[index]; + } + + return profile.Size switch + { + NativeContractInputSize.Tiny or NativeContractInputSize.Medium => ECCurve.Secp256k1.G, + NativeContractInputSize.Small or NativeContractInputSize.Large => ECCurve.Secp256r1.G, + _ => ECCurve.Secp256r1.G + }; + }, + profile => $"ECPoint({profile.Size})"); + + builder[typeof(ContractParameterType)] = new KnownFactory( + (_, _) => ContractParameterType.Integer, + _ => "ContractParameterType.Integer"); + + builder[typeof(CallFlags)] = new KnownFactory( + (_, _) => CallFlags.All, + _ => "CallFlags.All"); + + builder[typeof(NamedCurveHash)] = new KnownFactory( + (_, profile) => profile.Size switch + { + NativeContractInputSize.Tiny or NativeContractInputSize.Small => NamedCurveHash.secp256k1SHA256, + _ => NamedCurveHash.secp256r1SHA256 + }, + profile => profile.Size switch + { + NativeContractInputSize.Tiny or NativeContractInputSize.Small => "NamedCurveHash.secp256k1SHA256", + _ => "NamedCurveHash.secp256r1SHA256" + }); + + builder[typeof(Role)] = new KnownFactory( + (_, profile) => ResolveRole(profile.Size), + profile => $"Role.{ResolveRole(profile.Size)}"); + + builder[typeof(ByteString)] = new KnownFactory( + (_, profile) => new ByteString(CreateBytes(profile.ByteLength)), + profile => $"ByteString[{Math.Max(1, profile.ByteLength)}]"); + + builder[typeof(VMBuffer)] = new KnownFactory( + (_, profile) => new VMBuffer(CreateBytes(profile.ByteLength)), + profile => $"Buffer[{Math.Max(1, profile.ByteLength)}]"); + + builder[typeof(StackItem)] = new KnownFactory( + (_, profile) => new ByteString(CreateBytes(Math.Max(1, profile.ByteLength / 2))), + profile => $"StackItem(ByteString[{Math.Max(1, profile.ByteLength / 2)}])"); + + builder[typeof(VMArray)] = new KnownFactory( + (_, profile) => + { + var element = new ByteString(CreateBytes(Math.Max(1, profile.ByteLength / 4))); + return new VMArray(Enumerable.Repeat(element, Math.Max(1, profile.ElementCount / 2))); + }, + profile => $"Array(ByteString x{Math.Max(1, profile.ElementCount / 2)})"); + + builder[typeof(VMStruct)] = new KnownFactory( + (_, profile) => + { + var element = new ByteString(CreateBytes(Math.Max(1, profile.ByteLength / 4))); + return new VMStruct(Enumerable.Repeat(element, Math.Max(1, profile.ElementCount / 2))); + }, + profile => $"Struct(ByteString x{Math.Max(1, profile.ElementCount / 2)})"); + + builder[typeof(HashAlgorithm)] = new KnownFactory( + (_, _) => HashAlgorithm.SHA256, + _ => "HashAlgorithm.SHA256"); + + return builder.ToImmutable(); + } + + private static byte[] CreateBlsSerializedBytes(NativeContractInputProfile profile) + { + return profile.Size switch + { + NativeContractInputSize.Tiny => G1Affine.Generator.ToCompressed(), + NativeContractInputSize.Small => G2Affine.Generator.ToCompressed(), + NativeContractInputSize.Medium => Gt.Generator.ToArray(), + NativeContractInputSize.Large => Gt.Generator.Double().ToArray(), + _ => G1Affine.Generator.ToCompressed() + }; + } + + private static int ClampToInt(BigInteger value) + { + if (value > int.MaxValue) return int.MaxValue; + if (value < int.MinValue) return int.MinValue; + return (int)value; + } + + private static uint ClampToUInt(BigInteger value) + { + if (value < uint.MinValue) return uint.MinValue; + if (value > uint.MaxValue) return uint.MaxValue; + return (uint)value; + } + + private static long ClampToLong(BigInteger value) + { + if (value > long.MaxValue) return long.MaxValue; + if (value < long.MinValue) return long.MinValue; + return (long)value; + } + + private static ulong ClampToULong(BigInteger value) + { + if (value < 0) return 0; + if (value > ulong.MaxValue) return ulong.MaxValue; + return (ulong)value; + } + + private static byte[] CreateBytes(int length) + { + var size = Math.Max(1, length); + var data = new byte[size]; + for (int i = 0; i < size; i++) + data[i] = (byte)((i * 37 + 11) & 0xFF); + return data; + } + + private static UInt160 CreateUInt160(int seed) + { + Span buffer = stackalloc byte[UInt160.Length]; + for (int i = 0; i < buffer.Length; i++) + buffer[i] = (byte)(seed + i * 13); + return new UInt160(buffer); + } + + private static UInt256 CreateUInt256(int seed) + { + Span buffer = stackalloc byte[UInt256.Length]; + for (int i = 0; i < buffer.Length; i++) + buffer[i] = (byte)(seed + i * 7); + return new UInt256(buffer); + } + + private static string CreateDecimalString(NativeContractInputProfile profile) + { + int length = Math.Max(1, Math.Min(profile.ByteLength, 64)); + const string digits = "1234567890"; + StringBuilder sb = new(length); + for (int i = 0; i < length; i++) + sb.Append(digits[i % digits.Length]); + return sb.ToString(); + } + + private static string CreateBase64String(NativeContractInputProfile profile) + { + var bytes = CreateBytes(Math.Max(4, profile.ByteLength)); + return Convert.ToBase64String(bytes); + } + + private static string CreateBase64UrlString(NativeContractInputProfile profile) + { + var base64 = CreateBase64String(profile); + return base64.TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + + private static string CreateBase58String(NativeContractInputProfile profile) + { + var bytes = CreateBytes(Math.Max(4, profile.ByteLength)); + return Base58.Encode(bytes); + } + + private static string CreateBase58CheckString(NativeContractInputProfile profile) + { + var bytes = CreateBytes(Math.Max(4, profile.ByteLength)); + return Base58.Base58CheckEncode(bytes); + } + + private static byte[] CreateJsonBytes(NativeContractInputProfile profile) + { + var value = Math.Abs(ClampToInt(profile.IntegerMagnitude)); + var json = $"{{\"value\":{value}}}"; + return Encoding.UTF8.GetBytes(json); + } + + private static StackItem CreateJsonFriendlyStackItem(NativeContractInputProfile profile) + { + var count = Math.Clamp(profile.ElementCount, 1, 6); + List items = new(count + 1); + for (int i = 0; i < count; i++) + { + var token = CreateUtf8Token(profile, i); + items.Add(new ByteString(Encoding.UTF8.GetBytes(token))); + } + + if (profile.Size != NativeContractInputSize.Tiny) + { + var nested = new VMArray(new StackItem[] + { + new ByteString(Encoding.UTF8.GetBytes("bucket")), + new ByteString(Encoding.UTF8.GetBytes(profile.Name)) + }); + items.Add(nested); + } + + return new VMArray(items); + } + + private static ECPoint[] CreateRoleNodes(NativeContractBenchmarkContext context, NativeContractInputProfile profile) + { + ArgumentNullException.ThrowIfNull(context); + var committee = context.ProtocolSettings.StandbyCommittee; + var desired = DescribeRoleNodeCount(profile); + if (committee.Count == 0) + return new[] { ECCurve.Secp256r1.G }; + var actual = Math.Min(desired, Math.Min(committee.Count, 32)); + return committee.Take(actual).ToArray(); + } + + private static int DescribeRoleNodeCount(NativeContractInputProfile profile) + { + return Math.Clamp(Math.Max(1, profile.ElementCount), 1, 32); + } + + private static uint CreatePolicyMilliseconds(NativeContractBenchmarkContext context) + { + var value = context.ProtocolSettings.MillisecondsPerBlock; + if (value == 0) + value = 1; + if (value > PolicyContract.MaxMillisecondsPerBlock) + value = PolicyContract.MaxMillisecondsPerBlock; + return value; + } + + private static uint CreatePolicyAttributeFee(NativeContractInputProfile profile) + { + var baseline = (uint)(profile.ElementCount * 1_000 + 100); + if (baseline == 0) + baseline = 100; + if (baseline > PolicyContract.MaxAttributeFee) + baseline = PolicyContract.MaxAttributeFee; + return baseline; + } + + private static long CreatePolicyFeePerByte(NativeContractInputProfile profile) + { + var candidate = PolicyContract.DefaultFeePerByte + profile.ElementCount * 10; + if (candidate > 100_00000000L) + candidate = 100_00000000L; + return candidate; + } + + private static uint CreatePolicyExecFeeFactor(NativeContractInputProfile profile) + { + var candidate = PolicyContract.DefaultExecFeeFactor + (uint)Math.Max(1, profile.ElementCount); + if (candidate < 1) + candidate = 1; + if (candidate > PolicyContract.MaxExecFeeFactor) + candidate = PolicyContract.MaxExecFeeFactor; + return candidate; + } + + private static uint CreatePolicyStoragePrice(NativeContractInputProfile profile) + { + var candidate = PolicyContract.DefaultStoragePrice + (uint)Math.Max(1, profile.ByteLength); + if (candidate > PolicyContract.MaxStoragePrice) + candidate = PolicyContract.MaxStoragePrice; + return candidate; + } + + private static uint CreatePolicyMaxVub(NativeContractBenchmarkContext context, NativeContractInputProfile profile) + { + var baseValue = context.ProtocolSettings.MaxValidUntilBlockIncrement; + if (baseValue == 0) + baseValue = 1; + var spread = Math.Min(baseValue - 1, (uint)Math.Max(1, profile.ElementCount)); + var candidate = baseValue > spread ? baseValue - spread : 1u; + var traceable = context.ProtocolSettings.MaxTraceableBlocks; + if (traceable > 0 && candidate >= traceable) + candidate = traceable - 1; + if (candidate == 0) + candidate = 1; + if (candidate > PolicyContract.MaxMaxValidUntilBlockIncrement) + candidate = PolicyContract.MaxMaxValidUntilBlockIncrement; + return candidate; + } + + private static uint CreatePolicyMaxTraceableBlocks(NativeContractBenchmarkContext context, NativeContractInputProfile profile) + { + var current = context.ProtocolSettings.MaxTraceableBlocks; + if (current == 0) + current = PolicyContract.MaxMaxTraceableBlocks; + var decrement = (uint)Math.Max(1, profile.ElementCount); + var candidate = current > decrement ? current - decrement : current - 1; + var maxValid = Math.Max(1u, context.ProtocolSettings.MaxValidUntilBlockIncrement); + if (candidate <= maxValid) + candidate = maxValid + 1; + if (candidate > PolicyContract.MaxMaxTraceableBlocks) + candidate = PolicyContract.MaxMaxTraceableBlocks; + return candidate; + } + + private static string CreateOracleUrl(NativeContractInputProfile profile) + { + var slug = profile.Name.ToLowerInvariant(); + var url = $"https://oracle.neo/{slug}/{profile.ElementCount}"; + return url.Length > 128 ? url[..128] : url; + } + + private static string CreateOracleFilter(NativeContractInputProfile profile) + { + return profile.Size switch + { + NativeContractInputSize.Tiny => string.Empty, + NativeContractInputSize.Small => "$.result", + NativeContractInputSize.Medium => "$.data.items[0]", + _ => "$.data.items[1]" + }; + } + + private static string CreateOracleCallback(NativeContractInputProfile profile) + { + return profile.Size switch + { + NativeContractInputSize.Tiny => "onOracle", + NativeContractInputSize.Small => "oracleHandler", + NativeContractInputSize.Medium => "oracleResult", + _ => "oracleLarge" + }; + } + + private static StackItem CreateOracleUserData(NativeContractInputProfile profile) + { + var bytes = CreateBytes(Math.Min(profile.ByteLength, 64)); + return new ByteString(bytes); + } + + private static long CreateOracleGasBudget(NativeContractInputProfile profile) + { + long baseValue = 0_20000000; + long increment = (long)Math.Max(1, profile.ElementCount) * 1_0000000; + return baseValue + increment; + } + + private static long CreateOraclePrice(NativeContractInputProfile profile) + { + long baseValue = 50_000000; + long bump = (long)Math.Max(1, profile.ElementCount) * 100_0000; + return baseValue + bump; + } + + private static uint CreateNotaryLockHeight(NativeContractBenchmarkContext context) + { + var current = NativeContract.Ledger.CurrentIndex(context.StoreView); + var baseHeight = Math.Max(2u, current + 5); + var depositTill = context.GetSeededNotaryTill(); + return Math.Max(baseHeight, depositTill + 1); + } + + private static uint CreateNotaryMaxDelta(NativeContractBenchmarkContext context) + { + var min = (uint)ProtocolSettings.Default.ValidatorsCount; + var half = Math.Max(min + 1, context.ProtocolSettings.MaxValidUntilBlockIncrement / 2); + var candidate = min + 5; + if (candidate >= half) + candidate = half > min ? half - 1 : min; + return Math.Max(min, candidate); + } + + private static StackItem CreateNotaryPaymentData(NativeContractBenchmarkContext context, NativeContractInputProfile profile) + { + var receiver = context.GetAccount(profile.Size, 2); + var targetHeight = Math.Max(context.SeededLedgerHeight + 5, context.GetSeededNotaryTill() + 2); + return new VMArray(new StackItem[] + { + new ByteString(receiver.ToArray()), + new Integer(targetHeight) + }); + } + + private static StackItem CreateNeoCandidateData(NativeContractBenchmarkContext context) + { + var committee = context.ProtocolSettings.StandbyCommittee; + var pubkey = committee.Count > 0 ? committee[0] : ECCurve.Secp256r1.G; + return new ByteString(pubkey.EncodePoint(true)); + } + + private static byte[] CreateSerializedStackItemBytes(NativeContractInputProfile profile) + { + var stackItem = CreateJsonFriendlyStackItem(profile); + return BinarySerializer.Serialize(stackItem, ExecutionEngineLimits.Default); + } + + private static string CreateUtf8Token(NativeContractInputProfile profile, int index) + { + const string alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"; + var length = Math.Clamp(profile.ByteLength / 8, 6, 32); + var builder = new StringBuilder(profile.Name.Length + length + 8); + builder.Append(profile.Name); + builder.Append('_'); + builder.Append(index); + builder.Append('_'); + for (int i = 0; i < length; i++) + builder.Append(alphabet[(index + i) % alphabet.Length]); + return builder.ToString(); + } + + private static Role ResolveRole(NativeContractInputSize size) => size switch + { + NativeContractInputSize.Tiny => Role.StateValidator, + NativeContractInputSize.Small => Role.Oracle, + NativeContractInputSize.Medium => Role.NeoFSAlphabetNode, + NativeContractInputSize.Large => Role.P2PNotary, + _ => Role.StateValidator + }; + + private Transaction CreateBenchmarkTransaction(NativeContractInputProfile profile) + { + var script = profile.Size switch + { + NativeContractInputSize.Tiny => global::System.Array.Empty(), + NativeContractInputSize.Small => new byte[] { (byte)OpCode.RET }, + NativeContractInputSize.Medium => Enumerable.Repeat((byte)OpCode.NOP, profile.ElementCount).ToArray(), + NativeContractInputSize.Large => Enumerable.Repeat((byte)OpCode.NOP, profile.ElementCount * 2).ToArray(), + _ => global::System.Array.Empty() + }; + + return new Transaction + { + Version = 0, + Nonce = 1, + Script = script, + Signers = global::System.Array.Empty(), + Attributes = global::System.Array.Empty(), + Witnesses = global::System.Array.Empty(), + ValidUntilBlock = 0, + SystemFee = 0, + NetworkFee = 0 + }; + } + + private static (byte[] PubKey, byte[] Signature) CreateEcdsaVector(string privateKeyHex, string publicKeyHex, ECCurve curve, HashAlgorithm hash) + { + var privateKey = Convert.FromHexString(privateKeyHex); + var publicPoint = ECPoint.Parse(publicKeyHex, curve); + var signature = Crypto.Sign(s_ecdsaMessage, privateKey, curve, hash); + return (publicPoint.EncodePoint(false), signature); + } + + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkArtifacts.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkArtifacts.cs new file mode 100644 index 0000000000..f4f21752d5 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkArtifacts.cs @@ -0,0 +1,98 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkArtifacts.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.Json; +using Neo.SmartContract; +using Neo.SmartContract.Manifest; +using Neo.VM; +using System; +using System.IO; +using System.Text; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Produces shared NEF and manifest artifacts for benchmark scenarios. + /// + internal static class NativeContractBenchmarkArtifacts + { + public static NefFile CreateBenchmarkNef() + { + var nef = new NefFile + { + Compiler = "benchmark", + Source = "benchmark", + Tokens = Array.Empty(), + Script = new byte[] { (byte)OpCode.RET } + }; + nef.CheckSum = NefFile.ComputeChecksum(nef); + return nef; + } + + public static byte[] CreateBenchmarkNefBytes(NativeContractInputProfile profile) + { + return CreateBenchmarkNefBytes(); + } + + public static byte[] CreateBenchmarkNefBytes() + { + var nef = CreateBenchmarkNef(); + using MemoryStream ms = new(); + using (var writer = new BinaryWriter(ms, Encoding.UTF8, leaveOpen: true)) + { + nef.Serialize(writer); + } + return ms.ToArray(); + } + + public static ContractManifest CreateBenchmarkManifestDefinition(NativeContractInputProfile profile) + { + return CreateBenchmarkManifestDefinition(profile.Name); + } + + public static ContractManifest CreateBenchmarkManifestDefinition(string profileName) + { + return new ContractManifest + { + Name = $"Benchmark.{profileName}", + Groups = Array.Empty(), + SupportedStandards = Array.Empty(), + Abi = new ContractAbi + { + Events = Array.Empty(), + Methods = new[] + { + new ContractMethodDescriptor + { + Name = ContractBasicMethod.Deploy, + Parameters = new[] + { + new ContractParameterDefinition { Name = "data", Type = ContractParameterType.Any }, + new ContractParameterDefinition { Name = "update", Type = ContractParameterType.Boolean } + }, + ReturnType = ContractParameterType.Void, + Offset = 0, + Safe = false + } + } + }, + Permissions = new[] { ContractPermission.DefaultPermission }, + Trusts = WildcardContainer.Create(), + Extra = new JObject() + }; + } + + public static byte[] CreateBenchmarkManifestBytes(NativeContractInputProfile profile) + { + return CreateBenchmarkManifestDefinition(profile).ToJson().ToByteArray(indented: false); + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkCase.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkCase.cs new file mode 100644 index 0000000000..a08b2e57e7 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkCase.cs @@ -0,0 +1,91 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkCase.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Represents a single benchmark scenario targeting a native contract method. + /// + public sealed class NativeContractBenchmarkCase + { + public NativeContractBenchmarkCase( + NativeContract contract, + string contractName, + string methodName, + MethodInfo handler, + IReadOnlyList parameters, + bool requiresApplicationEngine, + bool requiresSnapshot, + NativeContractInputProfile profile, + string scenarioName, + string parameterSummary, + Func argumentFactory, + long cpuFee, + long storageFee, + CallFlags requiredCallFlags) + { + Contract = contract ?? throw new ArgumentNullException(nameof(contract)); + ContractName = contractName ?? throw new ArgumentNullException(nameof(contractName)); + MethodName = methodName ?? throw new ArgumentNullException(nameof(methodName)); + Handler = handler ?? throw new ArgumentNullException(nameof(handler)); + Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); + RequiresApplicationEngine = requiresApplicationEngine; + RequiresSnapshot = requiresSnapshot; + Profile = profile ?? throw new ArgumentNullException(nameof(profile)); + ScenarioName = scenarioName ?? throw new ArgumentNullException(nameof(scenarioName)); + ParameterSummary = parameterSummary ?? throw new ArgumentNullException(nameof(parameterSummary)); + ArgumentFactory = argumentFactory ?? throw new ArgumentNullException(nameof(argumentFactory)); + CpuFee = cpuFee; + StorageFee = storageFee; + RequiredCallFlags = requiredCallFlags; + } + + public NativeContract Contract { get; } + + public string ContractName { get; } + + public string MethodName { get; } + + public string MethodDisplayName => $"{ContractName}.{MethodName}"; + + public MethodInfo Handler { get; } + + public IReadOnlyList Parameters { get; } + + public bool RequiresApplicationEngine { get; } + + public bool RequiresSnapshot { get; } + + public NativeContractInputProfile Profile { get; } + + public string ScenarioName { get; } + + public string ParameterSummary { get; } + + public Func ArgumentFactory { get; } + + public long CpuFee { get; } + + public long StorageFee { get; } + + public CallFlags RequiredCallFlags { get; } + + public string UniqueId => $"{ContractName}:{MethodName}:{ScenarioName}:{Profile.Size}"; + + public override string ToString() => $"{MethodDisplayName}[{ScenarioName}/{Profile.Name}]"; + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkConfig.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkConfig.cs new file mode 100644 index 0000000000..e294256a5f --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkConfig.cs @@ -0,0 +1,83 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkConfig.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Validators; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// BenchmarkDotNet configuration tailored for native contract benchmarks. + /// + public sealed class NativeContractBenchmarkConfig : ManualConfig + { + public NativeContractBenchmarkConfig(NativeContractBenchmarkSuite suite) + { + AddColumn(new NativeContractInfoColumn("contract", "Contract", "Native contract name", c => c.ContractName)); + AddColumn(new NativeContractInfoColumn("method", "Method", "Native method name", c => c.MethodName)); + AddColumn(new NativeContractInfoColumn("profile", "Profile", "Input size profile", c => $"{c.Profile.Size}")); + AddColumn(new NativeContractInfoColumn("scenario", "Scenario", "Scenario label", c => c.ScenarioName)); + AddColumn(new NativeContractInfoColumn("parameters", "Inputs", "Generated parameter summary", c => c.ParameterSummary)); + AddColumn(new NativeContractInfoColumn("cpufee", "CpuFee", "Declared CPU fee", c => c.CpuFee.ToString())); + AddColumn(new NativeContractInfoColumn("storagefee", "StorageFee", "Declared storage fee", c => c.StorageFee.ToString())); + + AddLogger(ConsoleLogger.Unicode); + AddDiagnoser(MemoryDiagnoser.Default); + AddExporter(new NativeContractSummaryExporter(suite)); + AddExporter(MarkdownExporter.Default); + AddValidator(JitOptimizationsValidator.DontFailOnError); + AddColumnProvider(DefaultColumnProviders.Instance); + AddJob(CreateJob()); + SummaryStyle = SummaryStyle.Default.WithMaxParameterColumnWidth(60); + Options |= ConfigOptions.DisableOptimizationsValidator | ConfigOptions.KeepBenchmarkFiles; + } + + private static Job CreateJob() + { + return NativeContractBenchmarkOptions.Job switch + { + NativeContractBenchmarkJobMode.Quick => CreateQuickJob(), + NativeContractBenchmarkJobMode.Short => CreateShortJob(), + _ => CreateDefaultJob() + }; + } + + internal static Job CreateQuickJob() + { + return Job.Dry + .WithLaunchCount(1) + .WithWarmupCount(1) + .WithIterationCount(1) + .WithInvocationCount(1) + .WithUnrollFactor(1) + .WithId("QuickNative"); + } + + internal static Job CreateShortJob() + { + return Job.ShortRun + .WithId("ShortNative"); + } + + internal static Job CreateDefaultJob() + { + return Job.Default + .WithId("DefaultNative"); + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkContext.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkContext.cs new file mode 100644 index 0000000000..72221337f1 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkContext.cs @@ -0,0 +1,772 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkContext.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.Cryptography; +using Neo.Cryptography.ECC; +using Neo.Extensions; +using Neo.IO; +using Neo.Json; +using Neo.Ledger; +using Neo.Network.P2P; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Manifest; +using Neo.SmartContract.Native; +using Neo.VM; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Reflection; +using System.Text; +using System.Threading; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Provides access to shared runtime assets needed to invoke native contract methods. + /// + public sealed class NativeContractBenchmarkContext : IDisposable + { + private long _nonceCounter; + private readonly Signer[] _templateSigners; + private readonly UInt160[] _benchmarkAccounts; + private uint _seededNotaryTill; + private uint _seededLedgerHeight; + private bool _hasSeededNotaryTill; + private readonly UInt160 _callbackContractHash; + private readonly Dictionary _contractManagementCallers = new(); + private readonly KeyPair _notaryKeyPair; + private readonly UInt160 _notaryDepositAccount; + private BigInteger _neoRegisterPrice; + private UInt256 _seededLedgerTransactionHash = UInt256.Zero; + private const ulong OracleSeedRequestId = 0; + private static readonly byte[] s_notaryPrivateKey = Convert.FromHexString("0102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F20"); + private static readonly UInt160 s_notaryAttributeHash = Neo.SmartContract.Helper.GetContractHash(UInt160.Zero, 0, "Notary"); + private static readonly BigInteger s_notarySeedDepositAmount = NativeContract.GAS.Factor * 100; + + public NativeContractBenchmarkContext(NeoSystem system, ProtocolSettings protocolSettings) + { + System = system ?? throw new ArgumentNullException(nameof(system)); + ProtocolSettings = protocolSettings ?? throw new ArgumentNullException(nameof(protocolSettings)); + _benchmarkAccounts = BuildBenchmarkAccounts(protocolSettings); + _notaryKeyPair = new KeyPair(s_notaryPrivateKey); + _notaryDepositAccount = _benchmarkAccounts[0]; + _templateSigners = BuildDefaultSigners(_benchmarkAccounts); + SeedPolicyContractDefaults(); + SeedLedgerContractDefaults(); + SeedRoleManagementDefaults(); + SeedContractManagementDefaults(); + SeedNotaryDefaults(); + _callbackContractHash = SeedOracleCallbackContract(); + SeedOracleContractDefaults(); + SeedNeoTokenDefaults(); + } + + public NeoSystem System { get; } + + public ProtocolSettings ProtocolSettings { get; } + + public StoreCache GetSnapshot() => System.GetSnapshotCache(); + + public DataCache StoreView => System.StoreView; + + public ApplicationEngine CreateEngine(StoreCache snapshot, NativeContractBenchmarkCase benchmarkCase) + { + ArgumentNullException.ThrowIfNull(snapshot); + var tx = CreateBenchmarkTransaction(benchmarkCase); + return ApplicationEngine.Create( + trigger: TriggerType.Application, + container: tx, + snapshot: snapshot, + persistingBlock: System.GenesisBlock, + settings: ProtocolSettings, + gas: tx.SystemFee + tx.NetworkFee); + } + + private Transaction CreateBenchmarkTransaction(NativeContractBenchmarkCase benchmarkCase) + { + var nonce = unchecked((uint)Interlocked.Increment(ref _nonceCounter)); + List attributes = new(); + + var transaction = new Transaction + { + Version = 0, + Nonce = nonce, + SystemFee = 20_00000000, + NetworkFee = 1_00000000, + ValidUntilBlock = ProtocolSettings.MaxTraceableBlocks, + Signers = CloneSigners(_templateSigners), + Script = global::System.Array.Empty(), + Witnesses = global::System.Array.Empty() + }; + + if (RequiresOracleResponseAttribute(benchmarkCase)) + { + attributes.Add(new OracleResponse + { + Id = OracleSeedRequestId, + Code = OracleResponseCode.Success, + Result = Array.Empty() + }); + } + + if (RequiresNotaryAssistedAttribute(benchmarkCase)) + { + attributes.Add(new NotaryAssisted + { + NKeys = 1 + }); + } + + transaction.Attributes = attributes.ToArray(); + + return transaction; + } + + public UInt160 GetAccount(NativeContractInputSize size, int slot = 0) + { + int index = ((int)size + slot) % _benchmarkAccounts.Length; + return _benchmarkAccounts[index]; + } + + public UInt160 PrimaryBenchmarkAccount => _benchmarkAccounts[0]; + + public UInt160 CallbackContractHash => _callbackContractHash; + + public UInt160 NotaryDepositAccount => _notaryDepositAccount; + + public BigInteger NeoRegisterPrice => _neoRegisterPrice; + + public bool TryGetContractManagementCaller(NativeContractInputSize size, out UInt160 hash) + { + return _contractManagementCallers.TryGetValue(size, out hash); + } + + public uint SeededLedgerHeight => _seededLedgerHeight; + + public uint GetSeededNotaryTill() + { + if (!_hasSeededNotaryTill) + return Math.Max(ProtocolSettings.MaxTraceableBlocks, 100u); + return _seededNotaryTill; + } + + public byte[] CreateNotaryVerificationSignature(Transaction tx) + { + ArgumentNullException.ThrowIfNull(tx); + var hash = tx.GetSignData(ProtocolSettings.Network); + return Crypto.Sign(hash, _notaryKeyPair.PrivateKey); + } + + public void Dispose() + { + System.Dispose(); + } + + private static UInt160[] BuildBenchmarkAccounts(ProtocolSettings settings) + { + var committee = settings.StandbyCommittee; + int m = committee.Count - (committee.Count - 1) / 2; + var committeeAccount = Contract.CreateMultiSigRedeemScript(m, committee).ToScriptHash(); + + List accounts = + [ + committeeAccount + ]; + + accounts.AddRange(committee.Take(Math.Min(3, committee.Count)) + .Select(p => Contract.CreateSignatureRedeemScript(p).ToScriptHash())); + + accounts.Add(NativeContract.Notary.Hash); + accounts.Add(s_notaryAttributeHash); + accounts.Add(UInt160.Zero); + return accounts.Distinct().ToArray(); + } + + private static Signer[] BuildDefaultSigners(UInt160[] accounts) + { + return accounts.Select(account => new Signer + { + Account = account, + Scopes = account == NativeContract.Notary.Hash ? WitnessScope.None : WitnessScope.Global + }).ToArray(); + } + + private static Signer[] CloneSigners(Signer[] source) + { + if (source is null || source.Length == 0) + return global::System.Array.Empty(); + + var clones = new Signer[source.Length]; + for (int i = 0; i < source.Length; i++) + { + var signer = source[i]; + clones[i] = new Signer + { + Account = signer.Account, + Scopes = signer.Scopes, + AllowedContracts = signer.AllowedContracts is null ? null : signer.AllowedContracts.ToArray(), + AllowedGroups = signer.AllowedGroups is null ? null : signer.AllowedGroups.ToArray(), + Rules = signer.Rules is null ? null : signer.Rules.ToArray() + }; + } + + return clones; + } + + private static bool RequiresOracleResponseAttribute(NativeContractBenchmarkCase benchmarkCase) + { + return benchmarkCase is not null && + benchmarkCase.ContractName == nameof(OracleContract) && + string.Equals(benchmarkCase.MethodName, "Finish", StringComparison.OrdinalIgnoreCase); + } + + private static bool RequiresNotaryAssistedAttribute(NativeContractBenchmarkCase benchmarkCase) + { + if (benchmarkCase is null) + return false; + + if (benchmarkCase.ContractName != nameof(Notary)) + return false; + + return string.Equals(benchmarkCase.MethodName, "Verify", StringComparison.OrdinalIgnoreCase); + } + + private void SeedPolicyContractDefaults() + { + var policy = NativeContract.Policy; + using var snapshot = System.GetSnapshotCache(); + var seeded = false; + seeded |= EnsurePolicyValue(snapshot, policy, "_feePerByte", new StorageItem(PolicyContract.DefaultFeePerByte)); + seeded |= EnsurePolicyValue(snapshot, policy, "_execFeeFactor", new StorageItem(PolicyContract.DefaultExecFeeFactor)); + seeded |= EnsurePolicyValue(snapshot, policy, "_storagePrice", new StorageItem(PolicyContract.DefaultStoragePrice)); + seeded |= EnsurePolicyValue(snapshot, policy, "_millisecondsPerBlock", new StorageItem(ProtocolSettings.MillisecondsPerBlock)); + seeded |= EnsurePolicyValue(snapshot, policy, "_maxValidUntilBlockIncrement", new StorageItem(ProtocolSettings.MaxValidUntilBlockIncrement)); + seeded |= EnsurePolicyValue(snapshot, policy, "_maxTraceableBlocks", new StorageItem(ProtocolSettings.MaxTraceableBlocks)); + if (seeded) + snapshot.Commit(); + } + + private static bool EnsurePolicyValue(StoreCache snapshot, PolicyContract policy, string fieldName, StorageItem value) + { + var field = typeof(PolicyContract).GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {fieldName} on PolicyContract."); + var key = (StorageKey)field.GetValue(policy)!; + if (snapshot.Contains(key)) + return false; + snapshot.Add(key, value); + return true; + } + + private void SeedOracleContractDefaults() + { + var oracle = NativeContract.Oracle; + using var snapshot = System.GetSnapshotCache(); + var mutated = false; + + var priceKey = StorageKey.Create(oracle.Id, GetOraclePrefix("Prefix_Price")); + if (!snapshot.Contains(priceKey)) + { + snapshot.Add(priceKey, new StorageItem(0_50000000)); + mutated = true; + } + + var requestIdKey = StorageKey.Create(oracle.Id, GetOraclePrefix("Prefix_RequestId")); + var requestIdItem = new StorageItem(BigInteger.One); + if (snapshot.Contains(requestIdKey)) + { + snapshot.GetAndChange(requestIdKey).Value = requestIdItem.Value; + } + else + { + snapshot.Add(requestIdKey, requestIdItem); + mutated = true; + } + + var requestId = OracleSeedRequestId; + const string seedUrl = "https://oracle.neo/seed"; + var requestKey = StorageKey.Create(oracle.Id, GetOraclePrefix("Prefix_Request"), requestId); + if (!snapshot.Contains(requestKey)) + { + var userData = BinarySerializer.Serialize(Neo.VM.Types.StackItem.Null, ExecutionEngineLimits.Default); + var request = new OracleRequest + { + OriginalTxid = _seededLedgerTransactionHash == UInt256.Zero + ? UInt256.Zero + : _seededLedgerTransactionHash, + GasForResponse = 1_0000000, + Url = seedUrl, + Filter = string.Empty, + CallbackContract = _callbackContractHash, + CallbackMethod = "oracleHandler", + UserData = userData + }; + snapshot.Add(requestKey, StorageItem.CreateSealed(request)); + mutated = true; + } + + var urlHash = Crypto.Hash160(Encoding.UTF8.GetBytes(seedUrl)); + var listKey = StorageKey.Create(oracle.Id, GetOraclePrefix("Prefix_IdList"), urlHash); + if (!snapshot.Contains(listKey)) + { + var idListType = typeof(OracleContract).GetNestedType("IdList", BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to access OracleContract.IdList."); + var idList = (IInteroperable)Activator.CreateInstance(idListType)!; + idListType.GetMethod("Add")!.Invoke(idList, new object[] { requestId }); + snapshot.Add(listKey, StorageItem.CreateSealed(idList)); + mutated = true; + } + + if (mutated) + snapshot.Commit(); + } + + private static byte GetOraclePrefix(string name) + { + var field = typeof(OracleContract).GetField(name, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {name} on OracleContract."); + return (byte)field.GetValue(null)!; + } + + private void SeedLedgerContractDefaults() + { + var ledger = NativeContract.Ledger; + using var snapshot = System.GetSnapshotCache(); + var currentKey = (StorageKey)typeof(LedgerContract) + .GetField("_currentBlock", BindingFlags.Instance | BindingFlags.NonPublic)! + .GetValue(ledger)!; + + var block = CreateLedgerSeedBlock(); + _seededLedgerTransactionHash = block.Transactions.FirstOrDefault()?.Hash ?? UInt256.Zero; + var blockHashPrefix = GetLedgerPrefix("Prefix_BlockHash"); + var blockHashKey = StorageKey.Create(ledger.Id, blockHashPrefix, block.Index); + if (snapshot.Contains(blockHashKey)) + snapshot.Delete(blockHashKey); + snapshot.Add(blockHashKey, new StorageItem(block.Hash.ToArray())); + + var blockPrefix = GetLedgerPrefix("Prefix_Block"); + var trimmed = TrimmedBlock.Create(block); + var blockKey = StorageKey.Create(ledger.Id, blockPrefix, block.Hash); + if (snapshot.Contains(blockKey)) + snapshot.Delete(blockKey); + snapshot.Add(blockKey, new StorageItem(trimmed.ToArray())); + + var txPrefix = GetLedgerPrefix("Prefix_Transaction"); + foreach (var tx in block.Transactions) + { + var state = new TransactionState + { + BlockIndex = block.Index, + Transaction = tx, + State = VMState.HALT + }; + var txKey = StorageKey.Create(ledger.Id, txPrefix, tx.Hash); + if (snapshot.Contains(txKey)) + snapshot.Delete(txKey); + snapshot.Add(txKey, StorageItem.CreateSealed(state)); + } + + if (snapshot.Contains(currentKey)) + snapshot.Delete(currentKey); + snapshot.Add(currentKey, CreateHashIndexStorageItem(block.Hash, block.Index)); + _seededLedgerHeight = block.Index; + + snapshot.Commit(); + } + + private static byte GetLedgerPrefix(string name) + { + var field = typeof(LedgerContract).GetField(name, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {name} on LedgerContract."); + return (byte)field.GetValue(null)!; + } + + private static StorageItem CreateHashIndexStorageItem(UInt256 hash, uint index) + { + var type = typeof(LedgerContract).Assembly.GetType("Neo.SmartContract.Native.HashIndexState", throwOnError: true)!; + var instance = (IInteroperable)Activator.CreateInstance(type)!; + type.GetProperty("Hash")!.SetValue(instance, hash); + type.GetProperty("Index")!.SetValue(instance, index); + return StorageItem.CreateSealed(instance); + } + + private Block CreateLedgerSeedBlock() + { + var tx = CreateLedgerSeedTransaction(); + + var hashes = new[] { tx.Hash }; + var nextConsensus = Contract.GetBFTAddress( + ProtocolSettings.StandbyCommittee + .Take(Math.Max(1, ProtocolSettings.ValidatorsCount)) + .ToArray()); + + var header = new Header + { + PrevHash = UInt256.Zero, + MerkleRoot = MerkleTree.ComputeRoot(hashes), + Timestamp = System.GenesisBlock.Timestamp + ProtocolSettings.MillisecondsPerBlock, + Nonce = 1, + Index = 0, + PrimaryIndex = 0, + NextConsensus = nextConsensus, + Witness = new Witness + { + InvocationScript = ReadOnlyMemory.Empty, + VerificationScript = new[] { (byte)OpCode.PUSH1 } + } + }; + + return new Block + { + Header = header, + Transactions = new[] { tx } + }; + } + + private Transaction CreateLedgerSeedTransaction() + { + var signer = new Signer + { + Account = _benchmarkAccounts[0], + Scopes = WitnessScope.Global + }; + + return new Transaction + { + Version = 0, + Nonce = unchecked((uint)Interlocked.Increment(ref _nonceCounter)), + Script = new[] { (byte)OpCode.RET }, + SystemFee = 1_00000000, + NetworkFee = 1_0000000, + ValidUntilBlock = ProtocolSettings.MaxTraceableBlocks, + Signers = new[] { signer }, + Attributes = Array.Empty(), + Witnesses = new[] + { + new Witness + { + InvocationScript = ReadOnlyMemory.Empty, + VerificationScript = new[] { (byte)OpCode.PUSH1 } + } + } + }; + } + + private void SeedNotaryDefaults() + { + var notary = NativeContract.Notary; + using var snapshot = System.GetSnapshotCache(); + var mutated = false; + + var defaultDeltaField = typeof(Notary).GetField("DefaultMaxNotValidBeforeDelta", BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to access Notary.DefaultMaxNotValidBeforeDelta."); + var defaultDelta = (int)defaultDeltaField.GetValue(null)!; + var maxPrefix = GetNotaryPrefix("Prefix_MaxNotValidBeforeDelta"); + var maxKey = StorageKey.Create(notary.Id, maxPrefix); + if (!snapshot.Contains(maxKey)) + { + snapshot.Add(maxKey, new StorageItem((BigInteger)defaultDelta)); + mutated = true; + } + + var depositPrefix = GetNotaryPrefix("Prefix_Deposit"); + var account = _notaryDepositAccount; + var depositKey = StorageKey.Create(notary.Id, depositPrefix, account); + if (!snapshot.Contains(depositKey)) + { + var deposit = new Notary.Deposit + { + Amount = s_notarySeedDepositAmount, + Till = 0 + }; + snapshot.Add(depositKey, StorageItem.CreateSealed(deposit)); + _seededNotaryTill = deposit.Till; + _hasSeededNotaryTill = true; + EnsureGasLiquidity(snapshot, NativeContract.Notary.Hash, deposit.Amount); + mutated = true; + } + else + { + if (snapshot.TryGet(depositKey, out var item)) + { + _seededNotaryTill = item.GetInteroperable().Till; + _hasSeededNotaryTill = true; + } + } + + if (mutated) + snapshot.Commit(); + } + + private static byte GetNotaryPrefix(string name) + { + var field = typeof(Notary).GetField(name, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {name} on Notary."); + return (byte)field.GetValue(null)!; + } + + private void EnsureGasLiquidity(StoreCache snapshot, UInt160 account, BigInteger minimumBalance) + { + if (minimumBalance <= BigInteger.Zero) + return; + + var gas = NativeContract.GAS; + var accountPrefix = GetFungibleTokenPrefix(typeof(GasToken), "Prefix_Account"); + var totalPrefix = GetFungibleTokenPrefix(typeof(GasToken), "Prefix_TotalSupply"); + var accountKey = StorageKey.Create(gas.Id, accountPrefix, account); + var totalKey = StorageKey.Create(gas.Id, totalPrefix); + var totalItem = snapshot.GetAndChange(totalKey, () => new StorageItem(BigInteger.Zero)); + + if (!snapshot.TryGet(accountKey, out var accountItem)) + { + AccountState state = new() + { + Balance = minimumBalance + }; + snapshot.Add(accountKey, StorageItem.CreateSealed(state)); + totalItem.Add(minimumBalance); + return; + } + + var accountState = accountItem.GetInteroperable(); + if (accountState.Balance >= minimumBalance) + return; + + var delta = minimumBalance - accountState.Balance; + accountState.Balance = minimumBalance; + totalItem.Add(delta); + } + + private void SeedNeoTokenDefaults() + { + var neo = NativeContract.NEO; + using var snapshot = System.GetSnapshotCache(); + var mutated = false; + + var registerPriceField = typeof(NeoToken).GetField("_registerPrice", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to access NeoToken._registerPrice."); + var registerPriceKey = (StorageKey)registerPriceField.GetValue(neo)!; + _neoRegisterPrice = 10 * NativeContract.GAS.Factor; + var priceBytes = _neoRegisterPrice.ToByteArray(); + var priceItem = snapshot.Contains(registerPriceKey) + ? snapshot.GetAndChange(registerPriceKey) + : null; + if (priceItem is null) + { + snapshot.Add(registerPriceKey, new StorageItem(priceBytes)); + mutated = true; + } + else + { + priceItem.Value = priceBytes; + mutated = true; + } + + var accountPrefixField = typeof(NeoToken).BaseType? + .GetField("Prefix_Account", BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to access FungibleToken.Prefix_Account."); + var accountPrefix = (byte)accountPrefixField.GetValue(null)!; + var accountKey = StorageKey.Create(neo.Id, accountPrefix, PrimaryBenchmarkAccount); + if (!snapshot.Contains(accountKey)) + { + var state = new NeoToken.NeoAccountState + { + Balance = 100, + BalanceHeight = 0, + VoteTo = null, + LastGasPerVote = BigInteger.Zero + }; + snapshot.Add(accountKey, StorageItem.CreateSealed(state)); + mutated = true; + } + + EnsureGasLiquidity(snapshot, NativeContract.NEO.Hash, _neoRegisterPrice * 5); + + if (mutated) + snapshot.Commit(); + } + + private void SeedRoleManagementDefaults() + { + var roleManagement = NativeContract.RoleManagement; + using var snapshot = System.GetSnapshotCache(); + const uint index = 0; + var key = StorageKey.Create(roleManagement.Id, (byte)Role.P2PNotary, index); + if (snapshot.Contains(key)) + return; + + var nodeListType = typeof(RoleManagement).GetNestedType("NodeList", BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to access RoleManagement.NodeList."); + var nodeList = (IInteroperable)Activator.CreateInstance(nodeListType)! + ?? throw new InvalidOperationException("Unable to instantiate RoleManagement.NodeList."); + nodeListType.GetMethod("Add")!.Invoke(nodeList, new object[] { _notaryKeyPair.PublicKey }); + nodeListType.GetMethod("Sort")!.Invoke(nodeList, null); + snapshot.Add(key, StorageItem.CreateSealed(nodeList)); + snapshot.Commit(); + } + + private void SeedContractManagementDefaults() + { + var contractManagement = NativeContract.ContractManagement; + using var snapshot = System.GetSnapshotCache(); + var mutated = false; + + var minimumPrefix = GetContractManagementPrefix("Prefix_MinimumDeploymentFee"); + var minimumKey = StorageKey.Create(contractManagement.Id, minimumPrefix); + if (!snapshot.Contains(minimumKey)) + { + snapshot.Add(minimumKey, new StorageItem(10_00000000)); + mutated = true; + } + + var nextPrefix = GetContractManagementPrefix("Prefix_NextAvailableId"); + var nextKey = StorageKey.Create(contractManagement.Id, nextPrefix); + if (!snapshot.Contains(nextKey)) + { + snapshot.Add(nextKey, new StorageItem(10_000)); + mutated = true; + } + + var prefixContract = GetContractManagementPrefix("Prefix_Contract"); + var prefixContractHash = GetContractManagementPrefix("Prefix_ContractHash"); + const int ContractBaseId = 50_000; + + foreach (var profile in NativeContractInputProfiles.Default) + { + var manifest = NativeContractBenchmarkArtifacts.CreateBenchmarkManifestDefinition(profile); + var nef = NativeContractBenchmarkArtifacts.CreateBenchmarkNef(); + var hash = Neo.SmartContract.Helper.GetContractHash(UInt160.Zero, nef.CheckSum, manifest.Name); + var contractKey = StorageKey.Create(contractManagement.Id, prefixContract, hash); + if (!snapshot.Contains(contractKey)) + { + ContractState state = new() + { + Id = ContractBaseId + (int)profile.Size, + Hash = hash, + Manifest = manifest, + Nef = nef, + UpdateCounter = 0 + }; + snapshot.Add(contractKey, StorageItem.CreateSealed(state)); + snapshot.Add(StorageKey.Create(contractManagement.Id, prefixContractHash, state.Id), new StorageItem(hash.ToArray())); + mutated = true; + } + + _contractManagementCallers[profile.Size] = hash; + } + + if (mutated) + snapshot.Commit(); + } + + private UInt160 SeedOracleCallbackContract() + { + var contractManagement = NativeContract.ContractManagement; + using var snapshot = System.GetSnapshotCache(); + + var script = CreateOracleCallbackScript(); + var nef = new NefFile + { + Compiler = "benchmark", + Source = "benchmark", + Tokens = Array.Empty(), + Script = script + }; + nef.CheckSum = NefFile.ComputeChecksum(nef); + var manifest = CreateOracleCallbackManifest(); + var hash = Neo.SmartContract.Helper.GetContractHash(UInt160.Zero, nef.CheckSum, manifest.Name); + + var prefixContract = GetContractManagementPrefix("Prefix_Contract"); + var prefixContractHash = GetContractManagementPrefix("Prefix_ContractHash"); + var contractKey = StorageKey.Create(contractManagement.Id, prefixContract, hash); + if (!snapshot.Contains(contractKey)) + { + ContractState state = new() + { + Id = int.MaxValue - 2, + Hash = hash, + Manifest = manifest, + Nef = nef, + UpdateCounter = 0 + }; + snapshot.Add(contractKey, StorageItem.CreateSealed(state)); + snapshot.Add(StorageKey.Create(contractManagement.Id, prefixContractHash, state.Id), new StorageItem(hash.ToArray())); + snapshot.Commit(); + } + + return hash; + } + + private static byte[] CreateOracleCallbackScript() + { + var builder = new ScriptBuilder(); + builder.Emit(OpCode.DROP); + builder.Emit(OpCode.DROP); + builder.Emit(OpCode.DROP); + builder.Emit(OpCode.DROP); + builder.Emit(OpCode.RET); + return builder.ToArray(); + } + + private static ContractManifest CreateOracleCallbackManifest() + { + return new ContractManifest + { + Name = "Benchmark.Callback", + Groups = Array.Empty(), + SupportedStandards = Array.Empty(), + Abi = new ContractAbi + { + Events = Array.Empty(), + Methods = + [ + new ContractMethodDescriptor + { + Name = "oracleHandler", + Parameters = + [ + new ContractParameterDefinition { Name = "url", Type = ContractParameterType.String }, + new ContractParameterDefinition { Name = "userData", Type = ContractParameterType.Any }, + new ContractParameterDefinition { Name = "code", Type = ContractParameterType.Integer }, + new ContractParameterDefinition { Name = "result", Type = ContractParameterType.ByteArray } + ], + ReturnType = ContractParameterType.Void, + Offset = 0, + Safe = true + } + ] + }, + Permissions = new[] { ContractPermission.DefaultPermission }, + Trusts = WildcardContainer.Create(), + Extra = new JObject() + }; + } + + private static byte GetContractManagementPrefix(string name) + { + var field = typeof(ContractManagement).GetField(name, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {name} on ContractManagement."); + return (byte)field.GetValue(null)!; + } + + private static byte GetFungibleTokenPrefix(Type tokenType, string fieldName) + { + var baseType = tokenType.BaseType + ?? throw new InvalidOperationException($"Unable to access base type for {tokenType.Name}."); + var field = baseType.GetField(fieldName, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to access field {fieldName} on {baseType.Name}."); + return (byte)field.GetValue(null)!; + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkInvoker.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkInvoker.cs new file mode 100644 index 0000000000..a953573efe --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkInvoker.cs @@ -0,0 +1,238 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkInvoker.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using Neo.VM; +using System; +using System.Reflection; +using System.Runtime.ExceptionServices; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Executes a single benchmark case. + /// + public sealed class NativeContractBenchmarkInvoker + { + private static readonly object NullSentinel = new(); + private static readonly byte[] EntryScript = new[] { (byte)OpCode.RET }; + + private readonly NativeContractBenchmarkCase _case; + private readonly NativeContractBenchmarkContext _context; + private ContractState _contractState; + private static readonly PropertyInfo NativeCallingScriptHashProperty = + typeof(ExecutionContextState).GetProperty("NativeCallingScriptHash", BindingFlags.Instance | BindingFlags.NonPublic); + + public NativeContractBenchmarkInvoker(NativeContractBenchmarkCase benchmarkCase, NativeContractBenchmarkContext context) + { + _case = benchmarkCase ?? throw new ArgumentNullException(nameof(benchmarkCase)); + _context = context ?? throw new ArgumentNullException(nameof(context)); + } + + public object Invoke() + { + StoreCache snapshot = null; + ApplicationEngine engine = null; + object[] invocationArguments = null; + + try + { + var userArguments = _case.ArgumentFactory(_context) ?? System.Array.Empty(); + invocationArguments = PrepareInvocationArguments(userArguments, ref snapshot, ref engine); + ApplyCaseSpecificArgumentOverrides(invocationArguments, engine); + + var target = _case.Handler.IsStatic ? null : _case.Contract; + var result = _case.Handler.Invoke(target, invocationArguments); + result = UnwrapContractTask(result); + return result ?? NullSentinel; + } + catch (TargetInvocationException ex) when (ex.InnerException is not null) + { + var fault = engine?.FaultException; + if (fault is not null) + { + Console.Error.WriteLine($"[Invoker] Engine fault: {fault}"); + throw new InvalidOperationException($"Native contract faulted: {fault.Message}", fault); + } + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex.InnerException).Throw(); + throw; + } + catch (Exception ex) + { + var fault = engine?.FaultException; + if (fault is not null && !ReferenceEquals(fault, ex)) + { + Console.Error.WriteLine($"[Invoker] Engine fault: {fault}"); + throw new InvalidOperationException($"Native contract faulted: {fault.Message}", fault); + } + throw; + } + finally + { + DisposeIfNeeded(invocationArguments); + engine?.Dispose(); + snapshot?.Dispose(); + } + } + + private object[] PrepareInvocationArguments(object[] userArguments, ref StoreCache snapshot, ref ApplicationEngine engine) + { + if (_case.RequiresApplicationEngine) + { + snapshot = _context.GetSnapshot(); + engine = _context.CreateEngine(snapshot, _case); + EnsureExecutionContext(engine); + var allArgs = new object[userArguments.Length + 1]; + allArgs[0] = engine; + Array.Copy(userArguments, 0, allArgs, 1, userArguments.Length); + return allArgs; + } + + if (_case.RequiresSnapshot) + { + snapshot = _context.GetSnapshot(); + var allArgs = new object[userArguments.Length + 1]; + allArgs[0] = snapshot; + Array.Copy(userArguments, 0, allArgs, 1, userArguments.Length); + return allArgs; + } + + return userArguments; + } + + private void EnsureExecutionContext(ApplicationEngine engine) + { + if (engine is null) + return; + + var originalCache = engine.SnapshotCache; + engine.LoadScript(EntryScript, configureState: state => + { + state.CallFlags = CallFlags.All; + state.SnapshotCache = originalCache; + }); + + var contractState = _contractState ??= _case.Contract.GetContractState(_context.ProtocolSettings, 0); + var script = contractState.Script.ToArray(); + engine.LoadScript(script, configureState: state => + { + state.Contract = contractState; + state.ScriptHash = contractState.Hash; + state.CallFlags = GetEffectiveCallFlags(); + state.SnapshotCache = originalCache; + }); + + ApplyNativeCallingScriptOverrides(engine); + } + + private void DisposeIfNeeded(object[] arguments) + { + if (arguments is null) + return; + + var start = (_case.RequiresApplicationEngine || _case.RequiresSnapshot) ? 1 : 0; + for (int i = start; i < arguments.Length; i++) + { + if (arguments[i] is IDisposable disposable) + disposable.Dispose(); + } + } + + private static object UnwrapContractTask(object result) + { + if (result is null) + return null; + + var type = result.GetType(); + if (IsContractTask(type)) + return null; + + return result; + } + + private static bool IsContractTask(Type type) + { + while (type is not null) + { + if (type.FullName is "Neo.SmartContract.ContractTask" or "Neo.SmartContract.ContractTask`1") + return true; + type = type.BaseType; + } + + return false; + } + + private void ApplyCaseSpecificArgumentOverrides(object[] arguments, ApplicationEngine engine) + { + if (arguments is null) + return; + + if (_case.ContractName == nameof(Notary) && IsCaseInsensitiveMethod("Verify")) + { + var tx = engine?.ScriptContainer as Transaction + ?? throw new InvalidOperationException("Notary.Verify benchmarks require a transaction container."); + var signature = _context.CreateNotaryVerificationSignature(tx); + var signatureIndex = _case.RequiresApplicationEngine ? 1 : 0; + arguments[signatureIndex] = signature; + } + } + + private CallFlags GetEffectiveCallFlags() + { + if (_case.ContractName == nameof(ContractManagement) && + (IsCaseInsensitiveMethod("Deploy") || IsCaseInsensitiveMethod("Update"))) + { + return CallFlags.All; + } + + return _case.RequiredCallFlags; + } + + private void ApplyNativeCallingScriptOverrides(ApplicationEngine engine) + { + if (engine?.CurrentContext is null || NativeCallingScriptHashProperty is null) + return; + + var state = engine.CurrentContext.GetState(); + + if (_case.ContractName == nameof(ContractManagement) && + (IsCaseInsensitiveMethod("Update") || IsCaseInsensitiveMethod("Destroy"))) + { + if (!_context.TryGetContractManagementCaller(_case.Profile.Size, out var caller)) + throw new InvalidOperationException($"No seeded ContractManagement caller for {_case.Profile.Size}."); + NativeCallingScriptHashProperty.SetValue(state, caller); + } + + if (_case.ContractName == nameof(OracleContract) && IsCaseInsensitiveMethod("Request")) + { + NativeCallingScriptHashProperty.SetValue(state, _context.CallbackContractHash); + } + + if (_case.ContractName == nameof(Notary) && IsCaseInsensitiveMethod("OnNEP17Payment")) + { + NativeCallingScriptHashProperty.SetValue(state, NativeContract.GAS.Hash); + } + + if (_case.ContractName == nameof(NeoToken) && IsCaseInsensitiveMethod("OnNEP17Payment")) + { + NativeCallingScriptHashProperty.SetValue(state, NativeContract.GAS.Hash); + } + } + + private bool IsCaseInsensitiveMethod(string expected) + { + return string.Equals(_case.MethodName, expected, StringComparison.OrdinalIgnoreCase); + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkOptions.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkOptions.cs new file mode 100644 index 0000000000..5c1f2f1c60 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkOptions.cs @@ -0,0 +1,207 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkOptions.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; + +#nullable enable + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Reads environment-driven filters that control which native contract benchmark scenarios are materialised. + /// + internal static class NativeContractBenchmarkOptions + { + private sealed record Pattern(string Raw, Regex Regex) + { + public bool IsMatch(string value) => Regex.IsMatch(value); + } + + private sealed record Options( + IReadOnlyCollection AllowedSizes, + IReadOnlyCollection ContractFilters, + IReadOnlyCollection MethodFilters, + int? CaseLimit, + NativeContractBenchmarkJobMode Job, + IReadOnlyList Diagnostics); + + private static readonly object s_syncRoot = new(); + private static Lazy s_options = CreateLazy(); + + public static IReadOnlyList Diagnostics => s_options.Value.Diagnostics; + + public static NativeContractBenchmarkJobMode Job => s_options.Value.Job; + + public static bool IsSizeAllowed(NativeContractInputSize size) + { + var options = s_options.Value; + return options.AllowedSizes.Count == 0 || options.AllowedSizes.Contains(size); + } + + public static bool ShouldInclude(NativeContractBenchmarkCase benchmarkCase, out string? reason) + { + var options = s_options.Value; + + if (options.ContractFilters.Count > 0 && !options.ContractFilters.Any(pattern => pattern.IsMatch(benchmarkCase.ContractName))) + { + reason = $"Skipped {benchmarkCase.MethodDisplayName}: filtered by NEO_NATIVE_BENCH_CONTRACT."; + return false; + } + + if (options.MethodFilters.Count > 0 && !options.MethodFilters.Any(pattern => pattern.IsMatch(benchmarkCase.MethodName))) + { + reason = $"Skipped {benchmarkCase.MethodDisplayName}: filtered by NEO_NATIVE_BENCH_METHOD."; + return false; + } + + reason = null; + return true; + } + + public static IReadOnlyList ApplyLimit( + IReadOnlyList cases, + List diagnostics) + { + var options = s_options.Value; + if (options.CaseLimit is null || cases.Count <= options.CaseLimit.Value) + return cases; + + diagnostics.Add($"Limited native contract benchmarks to {options.CaseLimit.Value} case(s) via NEO_NATIVE_BENCH_LIMIT."); + return cases.Take(options.CaseLimit.Value).ToList(); + } + + public static void Reload() + { + lock (s_syncRoot) + { + s_options = CreateLazy(); + } + } + + private static Lazy CreateLazy() => new(Create, LazyThreadSafetyMode.ExecutionAndPublication); + + private static Options Create() + { + List diagnostics = []; + + var sizes = ParseSizes(Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_SIZES"), diagnostics); + var contractFilters = ParsePatterns(Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_CONTRACT"), diagnostics); + var methodFilters = ParsePatterns(Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_METHOD"), diagnostics); + int? limit = ParseLimit(Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_LIMIT"), diagnostics); + var job = ParseJob(Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_JOB"), diagnostics); + + return new Options(sizes, contractFilters, methodFilters, limit, job, diagnostics); + } + + private static IReadOnlyCollection ParseSizes(string? value, List diagnostics) + { + if (string.IsNullOrWhiteSpace(value)) + return Array.Empty(); + + HashSet sizes = new(); + foreach (var token in Split(value)) + { + if (Enum.TryParse(token, true, out var size)) + { + sizes.Add(size); + } + else + { + diagnostics.Add($"Ignored unknown value '{token}' in NEO_NATIVE_BENCH_SIZES. Accepted values: {string.Join(", ", Enum.GetNames())}."); + } + } + return new ReadOnlyCollection(sizes.ToList()); + } + + private static IReadOnlyCollection ParsePatterns(string? value, List diagnostics) + { + if (string.IsNullOrWhiteSpace(value)) + return Array.Empty(); + + List patterns = []; + foreach (var token in Split(value)) + { + try + { + var regex = WildcardToRegex(token); + patterns.Add(new Pattern(token, regex)); + } + catch (ArgumentException) + { + diagnostics.Add($"Ignored invalid wildcard '{token}' in benchmark filter."); + } + } + return new ReadOnlyCollection(patterns); + } + + private static int? ParseLimit(string? value, List diagnostics) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + + if (int.TryParse(value, out var limit) && limit > 0) + return limit; + + diagnostics.Add($"Ignored NEO_NATIVE_BENCH_LIMIT='{value}'. Expected a positive integer."); + return null; + } + + private static NativeContractBenchmarkJobMode ParseJob(string? value, List diagnostics) + { + if (string.IsNullOrWhiteSpace(value)) + return NativeContractBenchmarkJobMode.Default; + + var token = value.Trim(); + switch (token.ToLowerInvariant()) + { + case "quick": + case "fast": + diagnostics.Add("Using Quick job profile via NEO_NATIVE_BENCH_JOB."); + return NativeContractBenchmarkJobMode.Quick; + case "short": + case "ci": + diagnostics.Add("Using Short job profile via NEO_NATIVE_BENCH_JOB."); + return NativeContractBenchmarkJobMode.Short; + case "default": + case "full": + case "standard": + diagnostics.Add("Using Default job profile via NEO_NATIVE_BENCH_JOB."); + return NativeContractBenchmarkJobMode.Default; + default: + diagnostics.Add($"Ignored unknown job profile '{token}' in NEO_NATIVE_BENCH_JOB. Supported values: Quick, Short, Default."); + return NativeContractBenchmarkJobMode.Default; + } + } + + private static IEnumerable Split(string value) => + value.Split(new[] { ',', ';', ' ' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + private static Regex WildcardToRegex(string pattern) + { + var escaped = Regex.Escape(pattern) + .Replace("\\*", ".*") + .Replace("\\?", "."); + return new Regex($"^{escaped}$", RegexOptions.IgnoreCase | RegexOptions.Compiled); + } + } + + internal enum NativeContractBenchmarkJobMode + { + Default, + Short, + Quick + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkSuite.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkSuite.cs new file mode 100644 index 0000000000..bab1b1cedc --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractBenchmarkSuite.cs @@ -0,0 +1,154 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractBenchmarkSuite.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using Neo.SmartContract; +using Neo.SmartContract.Native; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Reflection; + +#nullable enable + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Discovers native contract methods and materialises benchmark cases. + /// + public sealed class NativeContractBenchmarkSuite : IDisposable + { + private static readonly FieldInfo MethodDescriptorsField = typeof(NativeContract).GetField("_methodDescriptors", BindingFlags.Instance | BindingFlags.NonPublic)!; + private static readonly Type MetadataType = typeof(NativeContract).Assembly.GetType("Neo.SmartContract.Native.ContractMethodMetadata", throwOnError: true)!; + private static readonly PropertyInfo MetadataName = MetadataType.GetProperty("Name", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataHandler = MetadataType.GetProperty("Handler", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataParameters = MetadataType.GetProperty("Parameters", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataNeedEngine = MetadataType.GetProperty("NeedApplicationEngine", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataNeedSnapshot = MetadataType.GetProperty("NeedSnapshot", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataCpuFee = MetadataType.GetProperty("CpuFee", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataStorageFee = MetadataType.GetProperty("StorageFee", BindingFlags.Instance | BindingFlags.Public)!; + private static readonly PropertyInfo MetadataCallFlags = MetadataType.GetProperty("RequiredCallFlags", BindingFlags.Instance | BindingFlags.Public)!; + + private readonly NativeContractBenchmarkContext _context; + private readonly ReadOnlyCollection _cases; + private readonly ReadOnlyCollection _diagnostics; + private readonly NativeContractArgumentGenerator _argumentGenerator = new(); + + private NativeContractBenchmarkSuite(NativeContractBenchmarkContext context) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + var discovery = DiscoverCases(); + _cases = discovery.Cases ?? new ReadOnlyCollection(Array.Empty()); + _diagnostics = discovery.Diagnostics ?? new ReadOnlyCollection(Array.Empty()); + } + + public IReadOnlyList Cases => _cases; + + public IReadOnlyList Diagnostics => _diagnostics; + + public NativeContractBenchmarkContext Context => _context; + + public static NativeContractBenchmarkSuite CreateDefault(string? configurationPath = null) + { + var configPath = configurationPath ?? "config.json"; + var protocol = ProtocolSettings.Load(configPath); + var system = new NeoSystem(protocol, (string?)null); + var context = new NativeContractBenchmarkContext(system, protocol); + return new NativeContractBenchmarkSuite(context); + } + + public NativeContractBenchmarkInvoker CreateInvoker(NativeContractBenchmarkCase benchmarkCase) + { + return new NativeContractBenchmarkInvoker(benchmarkCase, _context); + } + + private (ReadOnlyCollection Cases, ReadOnlyCollection Diagnostics) DiscoverCases() + { + var cases = new List(); + var diagnostics = new List(); + diagnostics.AddRange(NativeContractBenchmarkOptions.Diagnostics); + + foreach (var contract in NativeContract.Contracts.OrderBy(c => c.Name, StringComparer.Ordinal)) + { + if (MethodDescriptorsField.GetValue(contract) is not IEnumerable descriptors) + continue; + + foreach (var metadata in descriptors) + { + if (MetadataName.GetValue(metadata) is not string methodName || + MetadataHandler.GetValue(metadata) is not MethodInfo handler || + MetadataParameters.GetValue(metadata) is not InteropParameterDescriptor[] parameters || + MetadataNeedEngine.GetValue(metadata) is not bool requiresEngine || + MetadataNeedSnapshot.GetValue(metadata) is not bool requiresSnapshot || + MetadataCpuFee.GetValue(metadata) is not long cpuFee || + MetadataStorageFee.GetValue(metadata) is not long storageFee || + MetadataCallFlags.GetValue(metadata) is not CallFlags callFlags) + { + diagnostics.Add($"Skipped {contract.Name}: unable to read metadata for {metadata}"); + continue; + } + + foreach (var profile in NativeContractInputProfiles.Default) + { + if (!NativeContractBenchmarkOptions.IsSizeAllowed(profile.Size)) + { + diagnostics.Add($"Skipped {contract.Name}.{methodName} [{profile.Name}] due to NEO_NATIVE_BENCH_SIZES filter."); + continue; + } + + if (!_argumentGenerator.TryBuildArgumentFactory(contract, handler, parameters, profile, out var factory, out var summary, out var failure)) + { + diagnostics.Add($"Skipped {contract.Name}.{methodName} [{profile.Name}] : {failure}"); + continue; + } + + var scenarioName = $"{profile.Name}"; + var benchmarkCase = new NativeContractBenchmarkCase( + contract, + contract.Name, + methodName, + handler, + parameters, + requiresEngine, + requiresSnapshot, + profile, + scenarioName, + summary, + factory, + cpuFee, + storageFee, + callFlags); + + if (!NativeContractBenchmarkOptions.ShouldInclude(benchmarkCase, out var reason)) + { + diagnostics.Add(reason ?? $"Skipped {benchmarkCase.MethodDisplayName}: filtered by benchmark options."); + continue; + } + + cases.Add(benchmarkCase); + } + } + } + + var finalCases = NativeContractBenchmarkOptions.ApplyLimit(cases, diagnostics); + var readonlyCases = finalCases is List list + ? new ReadOnlyCollection(list) + : new ReadOnlyCollection(finalCases.ToList()); + return (readonlyCases, new ReadOnlyCollection(diagnostics)); + } + + public void Dispose() + { + _context.Dispose(); + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInfoColumn.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInfoColumn.cs new file mode 100644 index 0000000000..c337586eb1 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInfoColumn.cs @@ -0,0 +1,80 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractInfoColumn.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using System; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Custom column for exposing native contract metadata in BenchmarkDotNet tables. + /// + public sealed class NativeContractInfoColumn : IColumn + { + private readonly string _id; + private readonly string _columnName; + private readonly string _legend; + private readonly Func _selector; + + public NativeContractInfoColumn(string id, string columnName, string legend, Func selector) + { + _id = id ?? throw new ArgumentNullException(nameof(id)); + _columnName = columnName ?? throw new ArgumentNullException(nameof(columnName)); + _legend = legend ?? columnName; + _selector = selector ?? throw new ArgumentNullException(nameof(selector)); + } + + public string Id => _id; + + public string ColumnName => _columnName; + + public string Legend => _legend; + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => GetValue(benchmarkCase); + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(benchmarkCase); + + private string GetValue(BenchmarkCase benchmarkCase) + { + if (!TryGetCase(benchmarkCase, out var nativeCase)) + return "n/a"; + return _selector(nativeCase); + } + + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; + + public bool IsAvailable(Summary summary) => true; + + public UnitType UnitType => UnitType.Dimensionless; + + public ColumnCategory Category => ColumnCategory.Job; + + public int PriorityInCategory => 0; + + public bool AlwaysShow => true; + + public bool IsNumeric => false; + + private static bool TryGetCase(BenchmarkCase benchmarkCase, out NativeContractBenchmarkCase nativeCase) + { + if (benchmarkCase.Parameters["Case"] is NativeContractBenchmarkCase c) + { + nativeCase = c; + return true; + } + + nativeCase = null; + return false; + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInputProfile.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInputProfile.cs new file mode 100644 index 0000000000..c3efe2738c --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractInputProfile.cs @@ -0,0 +1,91 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractInputProfile.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Numerics; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Identifies the canonical size buckets that native contract benchmarks can target. + /// + public enum NativeContractInputSize + { + Tiny, + Small, + Medium, + Large + } + + /// + /// Describes a representative workload bucket for a native contract benchmark. + /// + /// Logical size bucket identifier. + /// Friendly name used when rendering benchmark cases. + /// Preferred byte payload size for byte/string based parameters. + /// Preferred collection length for array/map based parameters. + /// Representative magnitude for integer based parameters. + /// Human readable description of the workload bucket. + public sealed record NativeContractInputProfile( + NativeContractInputSize Size, + string Name, + int ByteLength, + int ElementCount, + BigInteger IntegerMagnitude, + string Description) + { + public override string ToString() => $"{Name} ({Size})"; + } + + /// + /// Provides the canonical set of input profiles consumed by the benchmark suite. + /// + public static class NativeContractInputProfiles + { + private static readonly ReadOnlyCollection s_defaultProfiles = + [ + new NativeContractInputProfile( + NativeContractInputSize.Tiny, + "Tiny", + ByteLength: 32, + ElementCount: 1, + IntegerMagnitude: new BigInteger(1_000), + Description: "Fits in a single stack item / small hash sized input."), + new NativeContractInputProfile( + NativeContractInputSize.Small, + "Small", + ByteLength: 256, + ElementCount: 4, + IntegerMagnitude: BigInteger.Parse("1000000000000000000000000"), + Description: "Representative for typical application inputs."), + new NativeContractInputProfile( + NativeContractInputSize.Medium, + "Medium", + ByteLength: 2048, + ElementCount: 16, + IntegerMagnitude: BigInteger.One << 256, + Description: "Stresses allocation-heavy paths and tree traversals."), + new NativeContractInputProfile( + NativeContractInputSize.Large, + "Large", + ByteLength: 4096, + ElementCount: 64, + IntegerMagnitude: BigInteger.One << 512, + Description: "Upper bound for native contract payload sizes.") + ]; + + /// + /// Gets the shared read-only collection of default input profiles. + /// + public static IReadOnlyCollection Default => s_defaultProfiles; + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractManualRunner.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractManualRunner.cs new file mode 100644 index 0000000000..9ff18063d6 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractManualRunner.cs @@ -0,0 +1,742 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractManualRunner.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.Json; + +#nullable enable + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Provides a lightweight fallback harness that executes native contract benchmark cases without BenchmarkDotNet. + /// + internal static class NativeContractManualRunner + { + private const string DefaultOutputDirectory = "BenchmarkDotNet.Artifacts/manual"; + + internal sealed record ManualRunnerArguments( + bool RunManualSuite, + string[] ForwardedArgs, + string? OutputOverride, + int? IterationOverride, + int? WarmupOverride, + bool? VerboseOverride, + string? ContractFilter, + string? MethodFilter, + string? SizeFilter, + string? LimitOverride, + string? JobOverride); + + internal sealed record NativeContractManualRunnerOptions( + string OutputDirectory, + int Iterations, + int WarmupIterations, + int TrimPercentage, + bool Verbose); + + private sealed record ManualResult( + NativeContractBenchmarkCase Case, + double? MeanMilliseconds, + double? StdDevMilliseconds, + double? MinMilliseconds, + double? MaxMilliseconds, + double? MedianMilliseconds, + double? Percentile95Milliseconds, + string? Error); + + private readonly struct SampleStatistics + { + public SampleStatistics(double mean, double stdDev, double min, double max, double median, double percentile95) + { + Mean = mean; + StdDev = stdDev; + Min = min; + Max = max; + Median = median; + Percentile95 = percentile95; + } + + public double Mean { get; } + public double StdDev { get; } + public double Min { get; } + public double Max { get; } + public double Median { get; } + public double Percentile95 { get; } + } + + public static ManualRunnerArguments ParseArguments(ReadOnlySpan args) + { + List forwarded = new(); + bool manual = false; + string? output = null; + int? iterations = null; + int? warmup = null; + bool? verbose = null; + string? contractFilter = null; + string? methodFilter = null; + string? sizeFilter = null; + string? limitOverride = null; + string? jobOverride = null; + + for (int i = 0; i < args.Length; i++) + { + var current = args[i]; + if (IsSwitch(current, "--native-manual-run")) + { + manual = true; + continue; + } + + if (IsSwitch(current, "--native-output")) + { + manual = true; + output = ReadValue(args, ref i, "--native-output"); + continue; + } + + if (IsSwitch(current, "--native-iterations")) + { + manual = true; + iterations = ParsePositiveInt(ReadValue(args, ref i, "--native-iterations"), "--native-iterations"); + continue; + } + + if (IsSwitch(current, "--native-warmup")) + { + manual = true; + warmup = ParseNonNegativeInt(ReadValue(args, ref i, "--native-warmup"), "--native-warmup"); + continue; + } + + if (IsSwitch(current, "--native-verbose")) + { + manual = true; + verbose = true; + continue; + } + + if (IsSwitch(current, "--native-contract")) + { + manual = true; + contractFilter = ReadValue(args, ref i, "--native-contract"); + continue; + } + + if (IsSwitch(current, "--native-method")) + { + manual = true; + methodFilter = ReadValue(args, ref i, "--native-method"); + continue; + } + + if (IsSwitch(current, "--native-sizes")) + { + manual = true; + sizeFilter = ReadValue(args, ref i, "--native-sizes"); + continue; + } + + if (IsSwitch(current, "--native-limit")) + { + manual = true; + limitOverride = ReadValue(args, ref i, "--native-limit"); + continue; + } + + if (IsSwitch(current, "--native-job")) + { + manual = true; + jobOverride = ReadValue(args, ref i, "--native-job"); + continue; + } + + forwarded.Add(current); + } + + return new ManualRunnerArguments( + manual, + forwarded.ToArray(), + output, + iterations, + warmup, + verbose, + contractFilter, + methodFilter, + sizeFilter, + limitOverride, + jobOverride); + } + + public static NativeContractManualRunnerOptions CreateOptions(ManualRunnerArguments arguments) + { + var profileToken = arguments.JobOverride + ?? Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_JOB") + ?? Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_PROFILE"); + var profile = ResolveProfile(profileToken); + + var iterations = arguments.IterationOverride + ?? TryParseEnvironmentInt("NEO_NATIVE_BENCH_ITERATIONS") + ?? profile.Iterations; + + var warmup = arguments.WarmupOverride + ?? TryParseEnvironmentInt("NEO_NATIVE_BENCH_WARMUP") + ?? profile.Warmup; + + var verbose = arguments.VerboseOverride + ?? ParseEnvironmentBool("NEO_NATIVE_BENCH_VERBOSE", defaultValue: false); + + var output = arguments.OutputOverride ?? DefaultOutputDirectory; + if (!Path.IsPathRooted(output)) + output = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, output)); + + return new NativeContractManualRunnerOptions( + output, + iterations, + warmup, + profile.TrimPercentage, + verbose); + } + + public static int Run(NativeContractManualRunnerOptions options, ManualRunnerArguments arguments) + { + using var scope = ApplyFilterOverrides(arguments); + using var suite = NativeContractBenchmarkSuite.CreateDefault(); + if (suite.Cases.Count == 0) + { + Console.Error.WriteLine("[ManualRunner] No native contract scenarios discovered. Check filter environment variables."); + DumpDiagnostics(suite.Diagnostics); + return 1; + } + + Console.WriteLine($"[ManualRunner] Executing {suite.Cases.Count} scenario(s) with {options.Iterations} iteration(s), {options.WarmupIterations} warmup pass(es), trim={options.TrimPercentage}%."); + + List results = new(suite.Cases.Count); + + foreach (var benchmarkCase in suite.Cases) + { + var invoker = suite.CreateInvoker(benchmarkCase); + try + { + for (int i = 0; i < options.WarmupIterations; i++) + invoker.Invoke(); + + var samples = new double[options.Iterations]; + for (int i = 0; i < options.Iterations; i++) + { + var sw = Stopwatch.StartNew(); + invoker.Invoke(); + sw.Stop(); + samples[i] = sw.Elapsed.TotalMilliseconds; + } + + var statistics = CalculateStatistics(samples, options.TrimPercentage); + var result = new ManualResult( + benchmarkCase, + statistics.Mean, + statistics.StdDev, + statistics.Min, + statistics.Max, + statistics.Median, + statistics.Percentile95, + null); + results.Add(result); + + if (options.Verbose) + { + Console.WriteLine($"[ManualRunner] {benchmarkCase.MethodDisplayName} [{benchmarkCase.Profile.Size}] " + + $"Mean={statistics.Mean:n3} ms | Median={statistics.Median:n3} ms | P95={statistics.Percentile95:n3} ms | StdDev={statistics.StdDev:n3} ms | Min={statistics.Min:n3} ms | Max={statistics.Max:n3} ms"); + } + } + catch (Exception ex) + { + var baseException = ex; + while (baseException.InnerException is not null) + baseException = baseException.InnerException; + + var message = $"{baseException.GetType().Name}: {baseException.Message}"; + Console.Error.WriteLine($"[ManualRunner] {benchmarkCase.MethodDisplayName} [{benchmarkCase.Profile.Size}] failed: {message}"); + if (options.Verbose && baseException.StackTrace is not null) + Console.Error.WriteLine(baseException.StackTrace); + results.Add(new ManualResult(benchmarkCase, null, null, null, null, null, null, message)); + } + } + + var summaryPath = WriteSummary(options, suite, results); + var jsonPath = WriteJsonSummary(options, suite, results); + var htmlPath = WriteHtmlSummary(options, suite, results); + Console.WriteLine($"[ManualRunner] Summary written to {summaryPath}"); + Console.WriteLine($"[ManualRunner] JSON report written to {jsonPath}"); + Console.WriteLine($"[ManualRunner] HTML report written to {htmlPath}"); + return 0; + } + + private static string WriteSummary( + NativeContractManualRunnerOptions options, + NativeContractBenchmarkSuite suite, + IReadOnlyCollection results) + { + Directory.CreateDirectory(options.OutputDirectory); + var path = Path.Combine(options.OutputDirectory, "manual-native-contract-summary.txt"); + + using var writer = new StreamWriter(path, false, Encoding.UTF8); + writer.WriteLine("Manual Native Contract Benchmark Summary"); + writer.WriteLine($"Generated at {DateTimeOffset.UtcNow:u}"); + writer.WriteLine($"Iterations per case: {options.Iterations} (warmup {options.WarmupIterations})"); + writer.WriteLine($"Discovered {suite.Cases.Count} scenario(s); executed {results.Count} scenario(s)."); + writer.WriteLine($"Filters -> Contract: {Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_CONTRACT") ?? "(none)"}, " + + $"Method: {Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_METHOD") ?? "(none)"}, " + + $"Sizes: {Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_SIZES") ?? "(all)"}."); + writer.WriteLine(); + + if (suite.Diagnostics.Count > 0) + { + writer.WriteLine("Diagnostics:"); + foreach (var diagnostic in suite.Diagnostics) + writer.WriteLine($" - {diagnostic}"); + writer.WriteLine(); + } + + var grouped = results + .GroupBy(r => r.Case.ContractName, StringComparer.Ordinal) + .OrderBy(group => group.Key, StringComparer.Ordinal); + + foreach (var contractGroup in grouped) + { + writer.WriteLine(contractGroup.Key); + foreach (var entry in contractGroup + .OrderBy(r => r.Case.MethodName, StringComparer.Ordinal) + .ThenBy(r => r.Case.Profile.Size)) + { + var caseInfo = entry.Case; + if (entry.Error is not null) + { + writer.WriteLine( + $" - {caseInfo.MethodName} [{caseInfo.Profile.Size}] FAILED: {entry.Error}"); + } + else + { + writer.WriteLine( + $" - {caseInfo.MethodName} [{caseInfo.Profile.Size}] Mean: {ToNanoseconds(entry.MeanMilliseconds!.Value)} ns | " + + $"Median: {ToNanoseconds(entry.MedianMilliseconds!.Value)} ns | P95: {ToNanoseconds(entry.Percentile95Milliseconds!.Value)} ns | " + + $"StdDev: {ToNanoseconds(entry.StdDevMilliseconds!.Value)} ns | CpuFee: {caseInfo.CpuFee} | StorageFee: {caseInfo.StorageFee} | {caseInfo.RequiredCallFlags}"); + } + } + + var (aggMean, aggMedian, aggP95) = AggregateContractMetrics(contractGroup); + writer.WriteLine( + $" Aggregate Mean: {ToNanoseconds(aggMean)} ns | Median: {ToNanoseconds(aggMedian)} ns | P95: {ToNanoseconds(aggP95)} ns"); + writer.WriteLine(); + } + + if (!grouped.Any()) + writer.WriteLine("No benchmark cases executed. Verify configuration and input filters."); + + var successful = results + .Where(r => r.MeanMilliseconds.HasValue) + .OrderByDescending(r => r.MeanMilliseconds!.Value) + .Take(10) + .ToList(); + + if (successful.Count > 0) + { + writer.WriteLine("Top scenarios by mean duration:"); + foreach (var entry in successful) + { + writer.WriteLine( + $" - {entry.Case.MethodDisplayName} [{entry.Case.Profile.Size}] " + + $"{ToNanoseconds(entry.MeanMilliseconds!.Value):N1} ns"); + } + } + + return path; + } + + private static string WriteJsonSummary( + NativeContractManualRunnerOptions options, + NativeContractBenchmarkSuite suite, + IReadOnlyCollection results) + { + Directory.CreateDirectory(options.OutputDirectory); + var path = Path.Combine(options.OutputDirectory, "manual-native-contract-summary.json"); + + var payload = new + { + generatedAt = DateTimeOffset.UtcNow, + iterations = options.Iterations, + warmupIterations = options.WarmupIterations, + trimPercentage = options.TrimPercentage, + diagnostics = suite.Diagnostics, + filters = new + { + contract = Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_CONTRACT"), + method = Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_METHOD"), + sizes = Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_SIZES"), + limit = Environment.GetEnvironmentVariable("NEO_NATIVE_BENCH_LIMIT") + }, + contractAggregates = results + .Where(r => r.MeanMilliseconds.HasValue) + .GroupBy(r => r.Case.ContractName) + .Select(g => + { + var (mean, median, p95) = AggregateContractMetrics(g); + return new + { + contract = g.Key, + caseCount = g.Count(), + meanNanoseconds = ToNanoseconds(mean), + medianNanoseconds = ToNanoseconds(median), + percentile95Nanoseconds = ToNanoseconds(p95) + }; + }) + .ToList(), + cases = results.Select(r => new + { + contract = r.Case.ContractName, + method = r.Case.MethodName, + profile = r.Case.Profile.Size.ToString(), + scenario = r.Case.ScenarioName, + cpuFee = r.Case.CpuFee, + storageFee = r.Case.StorageFee, + callFlags = r.Case.RequiredCallFlags.ToString(), + parameterSummary = r.Case.ParameterSummary, + meanNanoseconds = r.MeanMilliseconds.HasValue ? ToNanoseconds(r.MeanMilliseconds.Value) : (double?)null, + stdDevNanoseconds = r.StdDevMilliseconds.HasValue ? ToNanoseconds(r.StdDevMilliseconds.Value) : (double?)null, + minNanoseconds = r.MinMilliseconds.HasValue ? ToNanoseconds(r.MinMilliseconds.Value) : (double?)null, + maxNanoseconds = r.MaxMilliseconds.HasValue ? ToNanoseconds(r.MaxMilliseconds.Value) : (double?)null, + medianNanoseconds = r.MedianMilliseconds.HasValue ? ToNanoseconds(r.MedianMilliseconds.Value) : (double?)null, + percentile95Nanoseconds = r.Percentile95Milliseconds.HasValue ? ToNanoseconds(r.Percentile95Milliseconds.Value) : (double?)null, + error = r.Error + }) + }; + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions + { + WriteIndented = true + }); + File.WriteAllText(path, json, Encoding.UTF8); + return path; + } + + private static string WriteHtmlSummary( + NativeContractManualRunnerOptions options, + NativeContractBenchmarkSuite suite, + IReadOnlyCollection results) + { + Directory.CreateDirectory(options.OutputDirectory); + var path = Path.Combine(options.OutputDirectory, "manual-native-contract-summary.html"); + + using var writer = new StreamWriter(path, false, Encoding.UTF8); + writer.WriteLine(""); + writer.WriteLine(""); + writer.WriteLine(""); + writer.WriteLine("Native Contract Benchmarks"); + writer.WriteLine(""); + writer.WriteLine(""); + writer.WriteLine($"

Native Contract Benchmarks

"); + writer.WriteLine($"

Generated at {DateTimeOffset.UtcNow:u}. Iterations: {options.Iterations}, Warmup: {options.WarmupIterations}, Trim: {options.TrimPercentage}%.

"); + + var grouped = results + .GroupBy(r => r.Case.ContractName, StringComparer.Ordinal) + .OrderBy(g => g.Key, StringComparer.Ordinal); + + foreach (var contractGroup in grouped) + { + writer.WriteLine($"

{contractGroup.Key}

"); + writer.WriteLine(""); + writer.WriteLine("" + + "" + + ""); + + bool isGray = false; + var methodGroups = contractGroup + .GroupBy(r => r.Case.MethodName, StringComparer.Ordinal) + .OrderBy(g => g.Key, StringComparer.Ordinal); + + foreach (var methodGroup in methodGroups) + { + isGray = !isGray; + bool highlight = ShouldHighlightVariance(methodGroup); + + foreach (var entry in methodGroup.OrderBy(r => r.Case.Profile.Size)) + { + var styleParts = new List(); + if (isGray) + styleParts.Add("background:#f9f9f9;"); + if (highlight) + styleParts.Add("color:#b00020;font-weight:bold;"); + var rowStyle = styleParts.Count > 0 ? $" style=\"{string.Concat(styleParts)}\"" : string.Empty; + + if (entry.MeanMilliseconds.HasValue) + { + writer.WriteLine($"" + + $"" + + $"" + + $"" + + $"" + + $"" + + $"" + + $"" + + $"" + + "" + + ""); + } + else + { + writer.WriteLine($"" + + $"" + + $"" + + "" + + $"" + + ""); + } + } + } + writer.WriteLine("
MethodProfileMean (us)Median (us)P95 (us)StdDev (us)Min (us)Max (us)Status
{entry.Case.MethodName}{entry.Case.Profile.Size}{ToMicroseconds(entry.MeanMilliseconds.Value):n3}{ToMicroseconds((entry.MedianMilliseconds ?? entry.MeanMilliseconds).Value):n3}{ToMicroseconds((entry.Percentile95Milliseconds ?? entry.MeanMilliseconds).Value):n3}{ToMicroseconds(entry.StdDevMilliseconds!.Value):n3}{ToMicroseconds(entry.MinMilliseconds!.Value):n3}{ToMicroseconds(entry.MaxMilliseconds!.Value):n3}OK
{entry.Case.MethodName}{entry.Case.Profile.Size}n/a{entry.Error}
"); + } + + writer.WriteLine(""); + return path; + } + + // No engine-specific extraction needed currently. + + private static double ToNanoseconds(double milliseconds) => milliseconds * 1_000_000.0; + private static double ToMicroseconds(double milliseconds) => milliseconds * 1_000.0; + + private static void DumpDiagnostics(IReadOnlyList diagnostics) + { + foreach (var line in diagnostics) + Console.Error.WriteLine($" - {line}"); + } + + private static bool IsSwitch(string value, string option) => + string.Equals(value, option, StringComparison.OrdinalIgnoreCase); + + private static string ReadValue(ReadOnlySpan args, ref int index, string option) + { + if (index + 1 >= args.Length) + throw new ArgumentException($"Expected value after {option}."); + return args[++index]; + } + + private static int ParsePositiveInt(string value, string option) + { + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || parsed <= 0) + throw new ArgumentException($"{option} expects a positive integer value."); + return parsed; + } + + private static int ParseNonNegativeInt(string value, string option) + { + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) || parsed < 0) + throw new ArgumentException($"{option} expects a non-negative integer value."); + return parsed; + } + + private static int ParseEnvironmentInt(string name, int defaultValue, int min) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + return defaultValue; + + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed >= min) + return parsed; + + Console.Error.WriteLine($"[ManualRunner] Ignored invalid value '{value}' for {name}. Using {defaultValue}."); + return defaultValue; + } + + private static bool ParseEnvironmentBool(string name, bool defaultValue) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + return defaultValue; + + return value.Equals("1", StringComparison.OrdinalIgnoreCase) || + value.Equals("true", StringComparison.OrdinalIgnoreCase) || + value.Equals("yes", StringComparison.OrdinalIgnoreCase); + } + + private static int? TryParseEnvironmentInt(string name) + { + var value = Environment.GetEnvironmentVariable(name); + if (string.IsNullOrWhiteSpace(value)) + return null; + + return int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) + ? parsed + : null; + } + + private sealed record ManualRunnerProfile(string Name, int Iterations, int Warmup, int TrimPercentage); + + private static ManualRunnerProfile ResolveProfile(string? token) + { + var normalized = token?.Trim().ToLowerInvariant(); + return normalized switch + { + "quick" or "fast" => new ManualRunnerProfile("Quick", 5, 0, 0), + "short" or "ci" => new ManualRunnerProfile("Short", 15, 2, 5), + "thorough" or "full" => new ManualRunnerProfile("Thorough", 40, 5, 15), + "default" or "standard" => new ManualRunnerProfile("Balanced", 20, 3, 10), + _ => new ManualRunnerProfile("Balanced", 20, 3, 10) + }; + } + + private static SampleStatistics CalculateStatistics(double[] samples, int trimPercentage) + { + if (samples is null || samples.Length == 0) + return new SampleStatistics(0, 0, 0, 0, 0, 0); + + Array.Sort(samples); + double min = samples[0]; + double max = samples[^1]; + double median = samples.Length % 2 == 0 + ? (samples[samples.Length / 2 - 1] + samples[samples.Length / 2]) / 2.0 + : samples[samples.Length / 2]; + double percentile95 = samples[(int)Math.Floor(0.95 * (samples.Length - 1))]; + + int trimCount = 0; + if (trimPercentage > 0) + { + trimCount = (int)Math.Floor(samples.Length * (trimPercentage / 100.0)); + trimCount = Math.Min(trimCount, (samples.Length - 1) / 2); + } + + int usableLength = samples.Length - (trimCount * 2); + if (usableLength <= 0) + usableLength = samples.Length; + + int start = trimCount; + double sum = 0; + for (int i = start; i < start + usableLength; i++) + sum += samples[i]; + double mean = sum / usableLength; + + double variance = 0; + for (int i = start; i < start + usableLength; i++) + { + var delta = samples[i] - mean; + variance += delta * delta; + } + double stdDev = Math.Sqrt(variance / usableLength); + + return new SampleStatistics(mean, stdDev, min, max, median, percentile95); + } + + private static (double Mean, double Median, double Percentile95) AggregateContractMetrics(IEnumerable results) + { + var entries = results + .Where(r => r.MeanMilliseconds.HasValue) + .ToList(); + + if (entries.Count == 0) + return (0, 0, 0); + + double mean = entries.Average(r => r.MeanMilliseconds!.Value); + + double median = entries + .Select(r => r.MedianMilliseconds ?? r.MeanMilliseconds!.Value) + .Average(); + + double p95 = entries + .Select(r => r.Percentile95Milliseconds ?? r.MeanMilliseconds!.Value) + .Average(); + + return (mean, median, p95); + } + + private static bool ShouldHighlightVariance(IEnumerable methodResults) + { + var results = methodResults.ToList(); + var tiny = results.FirstOrDefault(r => r.Case.Profile.Size == NativeContractInputSize.Tiny); + var large = results.FirstOrDefault(r => r.Case.Profile.Size == NativeContractInputSize.Large); + if (tiny?.MeanMilliseconds is null || large?.MeanMilliseconds is null) + return false; + + var tinyMean = tiny.MeanMilliseconds.Value; + var largeMean = large.MeanMilliseconds.Value; + if (tinyMean <= 0 || largeMean <= 0) + return false; + + var ratio = tinyMean > largeMean ? tinyMean / largeMean : largeMean / tinyMean; + return ratio >= 2.0; + } + + private static IDisposable ApplyFilterOverrides(ManualRunnerArguments arguments) + { + Dictionary overrides = new(); + + void Capture(string? candidate, string key) + { + if (candidate is not null) + overrides[key] = candidate; + } + + Capture(arguments.ContractFilter, "NEO_NATIVE_BENCH_CONTRACT"); + Capture(arguments.MethodFilter, "NEO_NATIVE_BENCH_METHOD"); + Capture(arguments.SizeFilter, "NEO_NATIVE_BENCH_SIZES"); + Capture(arguments.LimitOverride, "NEO_NATIVE_BENCH_LIMIT"); + Capture(arguments.JobOverride, "NEO_NATIVE_BENCH_JOB"); + + if (overrides.Count == 0) + return NullScope.Instance; + + return new EnvOverrideScope(overrides); + } + + private sealed class EnvOverrideScope : IDisposable + { + private readonly Dictionary _previous = new(); + private bool _disposed; + + public EnvOverrideScope(Dictionary overrides) + { + foreach (var key in overrides.Keys) + _previous[key] = Environment.GetEnvironmentVariable(key); + + foreach (var (key, value) in overrides) + Environment.SetEnvironmentVariable(key, value); + + NativeContractBenchmarkOptions.Reload(); + } + + public void Dispose() + { + if (_disposed) + return; + + foreach (var (key, value) in _previous) + Environment.SetEnvironmentVariable(key, value); + + NativeContractBenchmarkOptions.Reload(); + _disposed = true; + } + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractSummaryExporter.cs b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractSummaryExporter.cs new file mode 100644 index 0000000000..be7bd957a7 --- /dev/null +++ b/benchmarks/Neo.Benchmarks/NativeContracts/NativeContractSummaryExporter.cs @@ -0,0 +1,157 @@ +// Copyright (C) 2015-2025 The Neo Project. +// +// NativeContractSummaryExporter.cs file belongs to the neo project and is free +// software distributed under the MIT software license, see the +// accompanying file LICENSE in the main directory of the +// repository or http://www.opensource.org/licenses/mit-license.php +// for more details. +// +// Redistribution and use in source and binary forms with or without +// modifications are permitted. + +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace Neo.Benchmarks.NativeContracts +{ + /// + /// Emits an aggregated report that highlights native contract performance distribution. + /// + public sealed class NativeContractSummaryExporter : IExporter + { + private readonly NativeContractBenchmarkSuite _suite; + + public NativeContractSummaryExporter(NativeContractBenchmarkSuite suite) + { + _suite = suite ?? throw new ArgumentNullException(nameof(suite)); + } + + public string Name => "native-contract-summary"; + + public string FileExtension => "txt"; + + public void ExportToLog(Summary summary, ILogger logger) + { + var payload = BuildSummary(summary); + logger?.WriteLine(payload); + } + + public IEnumerable ExportToFiles(Summary summary, ILogger logger) + { + var payload = BuildSummary(summary); + Directory.CreateDirectory(summary.ResultsDirectoryPath); + var path = Path.Combine(summary.ResultsDirectoryPath, "native-contract-summary.txt"); + File.WriteAllText(path, payload); + logger?.WriteLineInfo($"Native contract summary exported to {path}"); + yield return path; + } + + private string BuildSummary(Summary summary) + { + var builder = new StringBuilder(); + builder.AppendLine("Native Contract Benchmark Summary"); + builder.AppendLine($"Generated at {DateTimeOffset.UtcNow:u}"); + builder.AppendLine(); + + if (_suite.Diagnostics.Count > 0) + { + builder.AppendLine("Skipped Scenarios:"); + foreach (var diagnostic in _suite.Diagnostics) + builder.AppendLine($" - {diagnostic}"); + builder.AppendLine(); + } + + var grouped = new Dictionary>(StringComparer.Ordinal); + var executedCaseIds = new HashSet(StringComparer.Ordinal); + + foreach (var report in summary.Reports) + { + if (!TryGetCase(report.BenchmarkCase, out var nativeCase)) + continue; + + executedCaseIds.Add(nativeCase.UniqueId); + + if (!grouped.TryGetValue(nativeCase.ContractName, out var list)) + { + list = new List<(NativeContractBenchmarkCase, BenchmarkReport)>(); + grouped[nativeCase.ContractName] = list; + } + list.Add((nativeCase, report)); + } + + var measuredScenarioCount = 0; + foreach (var contract in grouped) + measuredScenarioCount += contract.Value.Count; + + builder.AppendLine($"Discovered {_suite.Cases.Count} scenario(s); executed {executedCaseIds.Count} scenario(s) using {DescribeJob(NativeContractBenchmarkOptions.Job)} job profile."); + builder.AppendLine($"BenchmarkDotNet produced {summary.Reports.Count()} report(s) covering {measuredScenarioCount} measured scenario(s) across {grouped.Count} contract(s)."); + builder.AppendLine(); + + foreach (var contractGroup in grouped.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + builder.AppendLine(contractGroup.Key); + + foreach (var entry in contractGroup.Value.OrderBy(p => p.Case.MethodName, StringComparer.Ordinal)) + { + var caseInfo = entry.Case; + var stats = entry.Report.ResultStatistics; + var meanText = stats is null ? "n/a" : FormatStatistic(stats.Mean); + var stdDevText = stats is null ? "n/a" : FormatStatistic(stats.StandardDeviation); + builder.AppendLine( + $" - {caseInfo.MethodName} [{caseInfo.Profile.Size}] " + + $"Mean: {meanText} ns | StdDev: {stdDevText} ns | " + + $"CpuFee: {caseInfo.CpuFee} | StorageFee: {caseInfo.StorageFee} | CallFlags: {caseInfo.RequiredCallFlags}"); + } + + var meanValues = contractGroup.Value + .Select(p => p.Report.ResultStatistics?.Mean) + .Where(v => v.HasValue && !double.IsNaN(v.Value) && !double.IsInfinity(v.Value)) + .Select(v => v.Value) + .ToList(); + var aggregateMeanText = meanValues.Count == 0 ? "n/a" : FormatStatistic(meanValues.Average()); + builder.AppendLine($" Aggregate mean: {aggregateMeanText} ns"); + builder.AppendLine($" Scenarios measured: {contractGroup.Value.Count}"); + builder.AppendLine(); + } + + if (grouped.Count == 0) + builder.AppendLine("No benchmark cases executed. Verify configuration and supported method coverage."); + + return builder.ToString(); + } + + private static string FormatStatistic(double value) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + return "n/a"; + return value.ToString("N1", CultureInfo.InvariantCulture); + } + + private static bool TryGetCase(BenchmarkCase benchmarkCase, out NativeContractBenchmarkCase nativeCase) + { + if (benchmarkCase.Parameters["Case"] is NativeContractBenchmarkCase c) + { + nativeCase = c; + return true; + } + + nativeCase = null; + return false; + } + + private static string DescribeJob(NativeContractBenchmarkJobMode jobMode) => jobMode switch + { + NativeContractBenchmarkJobMode.Quick => "Quick", + NativeContractBenchmarkJobMode.Short => "Short", + _ => "Default" + }; + } +} diff --git a/benchmarks/Neo.Benchmarks/Program.cs b/benchmarks/Neo.Benchmarks/Program.cs index 16255a17ee..e8a40e8912 100644 --- a/benchmarks/Neo.Benchmarks/Program.cs +++ b/benchmarks/Neo.Benchmarks/Program.cs @@ -10,6 +10,7 @@ // modifications are permitted. using BenchmarkDotNet.Running; +using Neo.Benchmarks.NativeContracts; // List all benchmarks: // dotnet run -c Release --framework [for example: net9.0] -- --list flat(or tree) @@ -20,4 +21,13 @@ // Run all benchmarks of a class: // dotnet run -c Release --framework [for example: net9.0] -- -f '*Class*' // More options: https://benchmarkdotnet.org/articles/guides/console-args.html -BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); +// dotnet run -c Release --framework net10.0 --project benchmarks/Neo.Benchmarks/Neo.Benchmarks.csproj -- --native-manual-run +var parsed = NativeContractManualRunner.ParseArguments(args); +if (parsed.RunManualSuite) +{ + var options = NativeContractManualRunner.CreateOptions(parsed); + Environment.ExitCode = NativeContractManualRunner.Run(options, parsed); + return; +} + +BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(parsed.ForwardedArgs); diff --git a/scripts/run-native-benchmarks.sh b/scripts/run-native-benchmarks.sh new file mode 100755 index 0000000000..7224b1573b --- /dev/null +++ b/scripts/run-native-benchmarks.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROJECT="$ROOT_DIR/benchmarks/Neo.Benchmarks/Neo.Benchmarks.csproj" +CONFIGURATION="${CONFIGURATION:-Release}" +FRAMEWORK="${FRAMEWORK:-net10.0}" +OUTPUT_DIR="${OUTPUT_DIR:-BenchmarkDotNet.Artifacts/manual}" + +contract_filter="" +method_filter="" +size_filter="" +limit_filter="" +job_filter="" +iterations="" +warmup="" +verbose=0 + +print_usage() { + cat <<'EOF' +Usage: scripts/run-native-benchmarks.sh [options] [-- ] + +Options: + -c, --contract Comma/space separated contract wildcard(s) (NEO_NATIVE_BENCH_CONTRACT) + -m, --method Method wildcard(s) (NEO_NATIVE_BENCH_METHOD) + -s, --sizes Size list e.g. Tiny,Small (NEO_NATIVE_BENCH_SIZES) + -l, --limit Maximum number of cases to execute (NEO_NATIVE_BENCH_LIMIT) + -j, --job Benchmark job profile: quick | short | default (NEO_NATIVE_BENCH_JOB) + --iterations Number of measured iterations (manual runner only) + --warmup Number of warmup passes (manual runner only) + --framework Target framework (default: net10.0) + --configuration Build configuration (default: Release) + --output Manual summary output directory (default: BenchmarkDotNet.Artifacts/manual) + --verbose Enable verbose per-case logging + -h, --help Show this message + +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -c|--contract) contract_filter="$2"; shift 2 ;; + -m|--method) method_filter="$2"; shift 2 ;; + -s|--sizes) size_filter="$2"; shift 2 ;; + -l|--limit) limit_filter="$2"; shift 2 ;; + -j|--job) job_filter="$2"; shift 2 ;; + --iterations) iterations="$2"; shift 2 ;; + --warmup) warmup="$2"; shift 2 ;; + --framework) FRAMEWORK="$2"; shift 2 ;; + --configuration) CONFIGURATION="$2"; shift 2 ;; + --output) OUTPUT_DIR="$2"; shift 2 ;; + --verbose) verbose=1; shift ;; + -h|--help) + print_usage + exit 0 + ;; + --) + shift + break + ;; + *) + echo "Unknown option: $1" >&2 + print_usage >&2 + exit 1 + ;; + esac +done + +extra_args=("$@") + +cmd=( + dotnet run + -c "$CONFIGURATION" + --framework "$FRAMEWORK" + --project "$PROJECT" + -- + --native-manual-run + --native-output "$OUTPUT_DIR" +) + +[[ -n "$contract_filter" ]] && cmd+=("--native-contract" "$contract_filter") +[[ -n "$method_filter" ]] && cmd+=("--native-method" "$method_filter") +[[ -n "$size_filter" ]] && cmd+=("--native-sizes" "$size_filter") +[[ -n "$limit_filter" ]] && cmd+=("--native-limit" "$limit_filter") +[[ -n "$job_filter" ]] && cmd+=("--native-job" "$job_filter") +[[ -n "$iterations" ]] && cmd+=("--native-iterations" "$iterations") +[[ -n "$warmup" ]] && cmd+=("--native-warmup" "$warmup") +[[ "$verbose" -eq 1 ]] && cmd+=("--native-verbose") +cmd+=("${extra_args[@]}") + +echo "[run-native-benchmarks] ${cmd[*]}" +exec "${cmd[@]}"