Skip to content

Commit 827ec00

Browse files
arianejasuwienasarianejasuwienas
authored
test: batch 2 conformity tests (#4110)
Signed-off-by: Mariusz Jasuwienas <[email protected]>
1 parent 35ff5b1 commit 827ec00

File tree

2 files changed

+115
-69
lines changed

2 files changed

+115
-69
lines changed

‎packages/server/tests/acceptance/conformityTests.spec.ts‎

Lines changed: 83 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,14 @@
11
// SPDX-License-Identifier: Apache-2.0
2-
// import {
3-
// JSONSchemaObject,
4-
// MethodObject,
5-
// MethodOrReference,
6-
// OpenrpcDocument,
7-
// } from '@open-rpc/meta-schema';
8-
// import { parseOpenRPCDocument } from '@open-rpc/schema-utils-js';
9-
// import { expect } from 'chai';
2+
import { JSONSchemaObject, MethodObject, MethodOrReference, OpenrpcDocument } from '@open-rpc/meta-schema';
3+
import { parseOpenRPCDocument } from '@open-rpc/schema-utils-js';
4+
import { expect } from 'chai';
105
import fs from 'fs';
116
import path from 'path';
7+
import WebSocket from 'ws';
128

13-
// import WebSocket from 'ws';
149
import openRpcData from '../../../../docs/openrpc.json';
15-
// import CallerContract from '../contracts/Caller.json';
16-
// import LogsContract from '../contracts/Logs.json';
10+
import CallerContract from '../contracts/Caller.json';
11+
import LogsContract from '../contracts/Logs.json';
1712
import {
1813
chainId,
1914
gasLimit,
@@ -26,9 +21,9 @@ import {
2621
setTransaction1559_2930AndBlockHash,
2722
setTransaction1559AndBlockHash,
2823
setTransaction2930AndBlockHash,
29-
// WS_RELAY_URL,
24+
WS_RELAY_URL,
3025
} from './data/conformity/utils/constants';
31-
// import { TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces';
26+
import { TestCase, TestCases, UpdateParamFunction } from './data/conformity/utils/interfaces';
3227
import { processFileContent, splitReqAndRes } from './data/conformity/utils/processors';
3328
import {
3429
createContractLegacyTransaction,
@@ -37,53 +32,74 @@ import {
3732
transaction1559_2930,
3833
transaction2930,
3934
} from './data/conformity/utils/transactions';
40-
import {
41-
getLatestBlockHash,
42-
// sendRequestToRelay,
43-
signAndSendRawTransaction,
44-
} from './data/conformity/utils/utils';
45-
// import { hasResponseFormatIssues, isResponseValid } from './data/conformity/utils/validations';
35+
import { getLatestBlockHash, sendRequestToRelay, signAndSendRawTransaction } from './data/conformity/utils/utils';
36+
import { getMissingKeys, isResponseValid } from './data/conformity/utils/validations';
4637

4738
const directoryPath = path.resolve(__dirname, '../../../../node_modules/execution-apis/tests');
4839
const overwritesDirectoryPath = path.resolve(__dirname, 'data/conformity/overwrites');
4940

50-
// let relayOpenRpcData: OpenrpcDocument;
51-
// (async () => {
52-
// relayOpenRpcData = await parseOpenRPCDocument(JSON.stringify(openRpcData));
53-
// })().catch((error) => console.error('Error parsing OpenRPC document:', error));
54-
55-
// const synthesizeTestCases = function (testCases: TestCases, updateParamIfNeeded: UpdateParamFunction) {
56-
// for (const testName in testCases) {
57-
// it(`${testName}`, async function () {
58-
// const isErrorStatusExpected: boolean =
59-
// (testCases[testName]?.status && testCases[testName].status != 200) ||
60-
// !!JSON.parse(testCases[testName].response).error;
61-
// const method = relayOpenRpcData.methods.find(
62-
// (m: MethodOrReference): m is MethodObject => 'name' in m && m.name === testName.split(' ')[0],
63-
// );
64-
// const schema: JSONSchemaObject | undefined =
65-
// method?.result && 'schema' in method.result && typeof method.result.schema === 'object'
66-
// ? method.result.schema
67-
// : undefined;
68-
// try {
69-
// const req = updateParamIfNeeded(testName, JSON.parse(testCases[testName].request));
70-
// const res = await sendRequestToRelay(RELAY_URL, req, false);
71-
// const isResFormatInvalid: boolean = hasResponseFormatIssues(res, JSON.parse(testCases[testName].response));
72-
//
73-
// if (schema && schema.pattern) {
74-
// const check = isResponseValid(schema, res);
75-
// expect(check).to.be.true;
76-
// }
77-
//
78-
// expect(isResFormatInvalid).to.be.false;
79-
// expect(isErrorStatusExpected).to.be.false;
80-
// } catch (e: any) {
81-
// expect(isErrorStatusExpected).to.be.true;
82-
// expect(e?.response?.status).to.equal(testCases[testName].status);
83-
// }
84-
// });
85-
// }
86-
// };
41+
let relayOpenRpcData: OpenrpcDocument;
42+
(async () => {
43+
relayOpenRpcData = await parseOpenRPCDocument(JSON.stringify(openRpcData));
44+
})().catch((error) => console.error('Error parsing OpenRPC document:', error));
45+
46+
/**
47+
* Determines whether a given test case is expected to return an error response.
48+
*
49+
* @param testCase - The test case to evaluate.
50+
* @returns {boolean} `true` if an error response is expected, otherwise `false`.
51+
*
52+
* @example
53+
* ```typescript
54+
* const tc = { status: 404, response: '{"error": "Not found"}' };
55+
* console.log(isErrorResponseExpected(tc)); // true
56+
* ```
57+
*/
58+
const isErrorResponseExpected = function (testCase: TestCase): boolean {
59+
return (testCase?.status && testCase.status != 200) || !!JSON.parse(testCase.response).error;
60+
};
61+
62+
/**
63+
* Retrieves the JSON schema object for the result of a given method name from the OpenRPC data.
64+
*
65+
* @param name - The name of the method to look up.
66+
* @returns {JSONSchemaObject | undefined} The method's result schema, or `undefined` if not found or invalid.
67+
*
68+
* @example
69+
* ```typescript
70+
* const schema = getMethodSchema("eth_getBalance");
71+
* console.log(schema); // JSON schema object or undefined
72+
* ```
73+
*/
74+
const getMethodSchema = function (name: string): JSONSchemaObject | undefined {
75+
const method = relayOpenRpcData.methods.find(
76+
(m: MethodOrReference): m is MethodObject => 'name' in m && m.name === name,
77+
);
78+
return method?.result && 'schema' in method.result && typeof method.result.schema === 'object'
79+
? method.result.schema
80+
: undefined;
81+
};
82+
83+
const synthesizeTestCases = function (testCases: TestCases, updateParamIfNeeded: UpdateParamFunction) {
84+
for (const testName in testCases) {
85+
it(`${testName}`, async function () {
86+
const isErrorStatusExpected = isErrorResponseExpected(testCases[testName]);
87+
const schema = getMethodSchema(testName.split(' ')[0]);
88+
try {
89+
const req = updateParamIfNeeded(testName, JSON.parse(testCases[testName].request));
90+
const res = await sendRequestToRelay(RELAY_URL, req, false);
91+
if (schema && schema.pattern) {
92+
const check = isResponseValid(schema, res);
93+
expect(check).to.be.true;
94+
}
95+
expect(isErrorStatusExpected).to.be.false;
96+
} catch (e: any) {
97+
expect(isErrorStatusExpected).to.be.true;
98+
expect(e?.response?.status).to.equal(testCases[testName].status);
99+
}
100+
});
101+
}
102+
};
87103

88104
/**
89105
* To run the Ethereum Execution API tests as defined in the repository ethereum/execution-apis, it’s necessary
@@ -152,7 +168,7 @@ describe('@api-conformity', async function () {
152168
//
153169
// These test suites must be un-skipped. The code requires refactoring to resolve the
154170
// static analysis issues before they can be re-enabled.
155-
/* describe.skip('@conformity-batch-2 Ethereum execution apis tests', async function () {
171+
describe('@conformity-batch-2 Ethereum execution apis tests', async function () {
156172
this.timeout(240 * 1000);
157173

158174
let existingBlockFilter: string;
@@ -222,7 +238,7 @@ describe('@api-conformity', async function () {
222238
synthesizeTestCases(TEST_CASES_BATCH_2, updateParamIfNeeded);
223239
});
224240

225-
describe.skip('@conformity-batch-3 Ethereum execution apis tests', async function () {
241+
describe('@conformity-batch-3 Ethereum execution apis tests', async function () {
226242
this.timeout(240 * 1000);
227243

228244
let txHash: any;
@@ -333,7 +349,12 @@ describe('@api-conformity', async function () {
333349
});
334350
await new Promise((r) => setTimeout(r, 500));
335351

336-
const hasMissingKeys: boolean = hasResponseFormatIssues(response, JSON.parse(testCases[testName].response));
352+
const hasMissingKeys =
353+
getMissingKeys({
354+
actual: response,
355+
expected: JSON.parse(testCases[testName].response),
356+
wildcards: [],
357+
}).length > 0;
337358
expect(hasMissingKeys).to.be.false;
338359
});
339360
}
@@ -343,7 +364,7 @@ describe('@api-conformity', async function () {
343364
});
344365
});
345366

346-
describe.skip('@conformity-batch-4 Ethereum execution apis tests', async function () {
367+
describe('@conformity-batch-4 Ethereum execution apis tests', async function () {
347368
this.timeout(240 * 1000);
348369

349370
let existingCallerContractAddress: string | null;
@@ -543,12 +564,12 @@ describe('@api-conformity', async function () {
543564
synthesizeTestCases(TEST_CASES_BATCH_4, updateParamIfNeeded);
544565
});
545566

546-
describe.skip('@conformity-batch-5 Ethereum execution apis tests', async function () {
567+
describe('@conformity-batch-5 Ethereum execution apis tests', async function () {
547568
this.timeout(240 * 1000);
548569
// eslint-disable-next-line @typescript-eslint/no-var-requires
549570
const TEST_CASES_BATCH_5 = require('./data/conformity-tests-batch-5.json');
550571

551572
const updateParamIfNeeded = (_testName: any, request: any) => request;
552573
synthesizeTestCases(TEST_CASES_BATCH_5, updateParamIfNeeded);
553-
});*/
574+
});
554575
});

‎packages/server/tests/acceptance/data/conformity/utils/validations.ts‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ addFormats(ajv);
2020
* Validates response format by comparing actual response against expected response structure.
2121
*
2222
* @param actualResponse - The actual response received from the API call
23-
* @param expectedResponse - The expected response structure to validate against (can be object, string, or ErrorResponse)
23+
* @param expectedResponse - The expected response structure to validate against (can be an object, string, or ErrorResponse)
2424
* @param wildcards - Array of property paths to ignore during validation (default: empty array)
2525
* @returns {boolean} Returns true if the response format has issues (validation failed), false if format is valid
2626
*
@@ -84,12 +84,11 @@ export function hasResponseFormatIssues(
8484
}
8585
return true;
8686
}
87-
88-
const actualResponseKeys = extractKeys(actualResponse as Record<string, unknown>);
89-
const expectedResponseKeys = extractKeys(parsedExpectedResponse as Record<string, unknown>);
90-
const filteredExpectedKeys = expectedResponseKeys.filter((key) => !wildcards.includes(key));
91-
const missingKeys = filteredExpectedKeys.filter((key) => !actualResponseKeys.includes(key));
92-
87+
const missingKeys = getMissingKeys({
88+
actual: actualResponse as Record<string, unknown>,
89+
expected: parsedExpectedResponse,
90+
wildcards,
91+
});
9392
if (missingKeys.length > 0) {
9493
console.log(`Missing keys in response: ${JSON.stringify(missingKeys)}`);
9594
return true;
@@ -98,6 +97,32 @@ export function hasResponseFormatIssues(
9897
return hasValuesMismatch(actualResponse, parsedExpectedResponse, wildcards);
9998
}
10099

100+
/**
101+
* Gets the list of expected keys that are missing from the actual response,
102+
* excluding any wildcard keys that should be ignored.
103+
*
104+
* @param response - The object containing:
105+
* - actual: the actual response object to check
106+
* - expected: the expected response object
107+
* - wildcards: a list of keys to ignore during comparison
108+
* @returns {string[]} - An array of keys that are expected but missing in the actual response
109+
*/
110+
export function getMissingKeys(response: {
111+
actual: Record<string, unknown>;
112+
expected: Record<string, unknown>;
113+
wildcards: string[];
114+
}): string[] {
115+
const skipKeys = new Set([...extractKeys(response.actual), ...response.wildcards]);
116+
return extractKeys(response.expected).filter(notIn(skipKeys));
117+
}
118+
119+
/**
120+
* Predicate factory that checks whether a given key is not in a provided Set.
121+
*
122+
* @param set - A Set of keys to exclude
123+
*/
124+
const notIn = (set: ReadonlySet<string>) => (k: string) => !set.has(k);
125+
101126
/**
102127
* Checks if the actual response is missing required error properties
103128
*

0 commit comments

Comments
 (0)