Skip to content

Commit b09c474

Browse files
MeldironMeldiron
authored
Merge pull request #69 from open-runtimes/feat-long-cold-start
Feat: TCP health check
2 parents ad015f7 + 5bb0185 commit b09c474

File tree

5 files changed

+163
-35
lines changed

5 files changed

+163
-35
lines changed

‎app/http.php‎

Lines changed: 68 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
require_once __DIR__ . '/../vendor/autoload.php';
44

55
use Appwrite\Runtimes\Runtimes;
6+
use OpenRuntimes\Executor\Validator\TCP;
67
use OpenRuntimes\Executor\Usage;
78
use Swoole\Process;
89
use Swoole\Runtime;
@@ -412,6 +413,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
412413
$secret = \bin2hex(\random_bytes(16));
413414

414415
$activeRuntimes->set($runtimeName, [
416+
'listening' => false,
415417
'name' => $runtimeName,
416418
'hostname' => $runtimeHostname,
417419
'created' => $startTime,
@@ -570,14 +572,10 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
570572
'duration' => $duration,
571573
]);
572574

573-
$activeRuntimes->set($runtimeName, [
574-
'name' => $runtimeName,
575-
'hostname' => $runtimeHostname,
576-
'created' => $startTime,
577-
'updated' => \microtime(true),
578-
'status' => 'Up ' . \round($duration, 2) . 's',
579-
'key' => $secret,
580-
]);
575+
$activeRuntime = $activeRuntimes->get($runtimeName);
576+
$activeRuntime['updated'] = \microtime(true);
577+
$activeRuntime['status'] = 'Up ' . \round($duration, 2) . 's';
578+
$activeRuntimes->set($runtimeName, $activeRuntime);
581579
} catch (Throwable $th) {
582580
$error = $th->getMessage() . $output;
583581

@@ -734,7 +732,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
734732
'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname()
735733
]);
736734

737-
$coldStartDuration = 0;
735+
$prepareStart = \microtime(true);
738736

739737
// Prepare runtime
740738
if (!$activeRuntimes->exists($runtimeName)) {
@@ -793,7 +791,12 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
793791
};
794792

795793
// Prepare runtime
796-
for ($i = 0; $i < 10; $i++) {
794+
while (true) {
795+
// If timeout is passed, stop and return error
796+
if (\microtime(true) - $prepareStart >= $timeout) {
797+
throw new Exception('Function timed out during preparation.', 400);
798+
}
799+
797800
['errNo' => $errNo, 'error' => $error, 'statusCode' => $statusCode, 'executorResponse' => $executorResponse] = \call_user_func($sendCreateRuntimeRequest);
798801

799802
if ($errNo === 0) {
@@ -806,39 +809,44 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
806809
$error = $body['message'];
807810
throw new Exception('An internal curl error has occurred while starting runtime! Error Msg: ' . $error, 500);
808811
} else {
809-
$coldStartDuration = \floatval($body['duration']);
810812
break;
811813
}
812814
} elseif ($errNo !== 111) { // Connection Refused - see https://openswoole.com/docs/swoole-error-code
813815
throw new Exception('An internal curl error has occurred while starting runtime! Error Msg: ' . $error, 500);
814816
}
815817

816-
if ($i === 9) {
817-
throw new Exception('An internal curl error has occurred while starting runtime! Error Msg: ' . $error, 500);
818-
}
819-
818+
// Wait 0.5s and check again
820819
\usleep(500000);
821820
}
822821
}
823822

823+
// Lower timeout by time it took to prepare container
824+
$timeout -= (\microtime(true) - $prepareStart);
825+
824826
// Update swoole table
825827
$runtime = $activeRuntimes->get($runtimeName) ?? [];
826828
$runtime['updated'] = \time();
827829
$activeRuntimes->set($runtimeName, $runtime);
828830

829831
// Ensure runtime started
830-
for ($i = 0; $i < 10; $i++) {
831-
if ($activeRuntimes->get($runtimeName)['status'] !== 'pending') {
832-
break;
832+
$launchStart = \microtime(true);
833+
while (true) {
834+
// If timeout is passed, stop and return error
835+
if (\microtime(true) - $launchStart >= $timeout) {
836+
throw new Exception('Function timed out during launch.', 400);
833837
}
834838

835-
if ($i === 9) {
836-
throw new Exception('Runtime failed to launch in allocated time.', 500);
839+
if ($activeRuntimes->get($runtimeName)['status'] !== 'pending') {
840+
break;
837841
}
838842

843+
// Wait 0.5s and check again
839844
\usleep(500000);
840845
}
841846

847+
// Lower timeout by time it took to launch container
848+
$timeout -= (\microtime(true) - $launchStart);
849+
842850
// Ensure we have secret
843851
$runtime = $activeRuntimes->get($runtimeName);
844852
$hostname = $runtime['hostname'];
@@ -847,12 +855,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
847855
throw new Exception('Runtime secret not found. Please re-create the runtime.', 500);
848856
}
849857

850-
$startTime = \microtime(true);
851-
852-
$executeV2 = function () use ($variables, $payload, $secret, $hostname, &$startTime, $timeout): array {
853-
// Restart execution timer to not could failed attempts
854-
$startTime = \microtime(true);
855-
858+
$executeV2 = function () use ($variables, $payload, $secret, $hostname, $timeout): array {
856859
$statusCode = 0;
857860
$errNo = -1;
858861
$executorResponse = '';
@@ -925,10 +928,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
925928
];
926929
};
927930

928-
$executeV3 = function () use ($path, $method, $headers, $payload, $secret, $hostname, &$startTime, $timeout): array {
929-
// Restart execution timer to not could failed attempts
930-
$startTime = \microtime(true);
931-
931+
$executeV3 = function () use ($path, $method, $headers, $payload, $secret, $hostname, $timeout): array {
932932
$statusCode = 0;
933933
$errNo = -1;
934934
$executorResponse = '';
@@ -959,11 +959,11 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
959959

960960
return $len;
961961
});
962-
\curl_setopt($ch, CURLOPT_TIMEOUT, $timeout + 1); // Gives extra 1s after safe timeout to recieve response
963-
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
962+
\curl_setopt($ch, CURLOPT_TIMEOUT, $timeout + 5); // Gives extra 5s after safe timeout to recieve response
963+
\curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5);
964964

965965
$headers['x-open-runtimes-secret'] = $secret;
966-
$headers['x-open-runtimes-timeout'] = $timeout;
966+
$headers['x-open-runtimes-timeout'] = \max(\intval($timeout), 1);
967967
$headersArr = [];
968968
foreach ($headers as $key => $value) {
969969
$headersArr[] = $key . ': ' . $value;
@@ -1017,6 +1017,40 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
10171017
];
10181018
};
10191019

1020+
// From here we calculate billable duration of execution
1021+
$startTime = \microtime(true);
1022+
1023+
$listening = $runtime['listening'];
1024+
1025+
if (!$listening) {
1026+
// Wait for cold-start to finish (app listening on port)
1027+
$pingStart = \microtime(true);
1028+
$validator = new TCP();
1029+
while (true) {
1030+
// If timeout is passed, stop and return error
1031+
if (\microtime(true) - $pingStart >= $timeout) {
1032+
throw new Exception('Function timed out during cold start.', 400);
1033+
}
1034+
1035+
$online = $validator->isValid($hostname . ':' . 3000);
1036+
if ($online) {
1037+
break;
1038+
}
1039+
1040+
// Wait 0.5s and check again
1041+
\usleep(500000);
1042+
}
1043+
1044+
// Update swoole table
1045+
$runtime = $activeRuntimes->get($runtimeName);
1046+
$runtime['listening'] = true;
1047+
$activeRuntimes->set($runtimeName, $runtime);
1048+
1049+
// Lower timeout by time it took to cold-start
1050+
$timeout -= (\microtime(true) - $pingStart);
1051+
}
1052+
1053+
10201054
// Execute function
10211055
for ($i = 0; $i < 10; $i++) {
10221056
$executionRequest = $version === 'v3' ? $executeV3 : $executeV2;
@@ -1056,7 +1090,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
10561090
'body' => $body,
10571091
'logs' => \mb_strcut($logs, 0, 1000000), // Limit to 1MB
10581092
'errors' => \mb_strcut($errors, 0, 1000000), // Limit to 1MB
1059-
'duration' => $duration + $coldStartDuration,
1093+
'duration' => $duration,
10601094
'startTime' => $startTime,
10611095
];
10621096

‎src/Executor/Validator/TCP.php‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
<?php
2+
3+
namespace OpenRuntimes\Executor\Validator;
4+
5+
use Utopia\Http\Validator;
6+
7+
/**
8+
* TCP ping validator
9+
*
10+
* Validate that a port is open on an IP address.
11+
*/
12+
class TCP extends Validator
13+
{
14+
public function __construct(protected float $timeout = 10)
15+
{
16+
}
17+
18+
public function getDescription(): string
19+
{
20+
return 'Host is unreachable, or port is not open.';
21+
}
22+
23+
public function isArray(): bool
24+
{
25+
return false;
26+
}
27+
28+
public function getType(): string
29+
{
30+
return self::TYPE_STRING;
31+
}
32+
public function isValid(mixed $value): bool
33+
{
34+
$value = \strval($value);
35+
[ $ip, $port ] = \explode(':', $value);
36+
$port = \intval($port);
37+
38+
if (empty($port) || empty($ip)) {
39+
return false;
40+
}
41+
42+
// TCP Ping
43+
$errorCode = null;
44+
$errorMessage = "";
45+
$socket = \fsockopen($ip, $port, $errorCode, $errorMessage, $this->timeout);
46+
47+
if (!$socket) {
48+
return false;
49+
} else {
50+
\fclose($socket);
51+
return true;
52+
}
53+
}
54+
}

‎tests/ExecutorTest.php‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,22 @@ public function provideScenarios(): array
340340
$this->assertStringContainsString('Invalid response. This usually means too large logs or errors', $response['body']['message']);
341341
}
342342
],
343+
[
344+
'image' => 'openruntimes/node:v3-18.0',
345+
'entrypoint' => 'index.js',
346+
'folder' => 'node-long-coldstart',
347+
'version' => 'v3',
348+
'startCommand' => 'cp /tmp/code.tar.gz /mnt/code/code.tar.gz && nohup helpers/start.sh "pm2 start src/server.js --no-daemon"',
349+
'buildCommand' => 'tar -zxf /tmp/code.tar.gz -C /mnt/code && helpers/build.sh "npm i"',
350+
'assertions' => function ($response) {
351+
$this->assertEquals(200, $response['headers']['status-code']);
352+
$this->assertEquals(200, $response['body']['statusCode']);
353+
$this->assertEquals('1836311903', $response['body']['body']);
354+
$this->assertGreaterThan(10, $response['body']['duration']); // This is unsafe but important. If its flaky, inform @Meldiron
355+
$this->assertEmpty($response['body']['logs']);
356+
$this->assertEmpty($response['body']['errors']);
357+
}
358+
],
343359
];
344360
}
345361

@@ -384,7 +400,8 @@ public function testScenarios(string $image, string $entrypoint, string $folder,
384400
'entrypoint' => $entrypoint,
385401
'image' => $image,
386402
'version' => $version,
387-
'runtimeEntrypoint' => $startCommand
403+
'runtimeEntrypoint' => $startCommand,
404+
'timeout' => 45
388405
]);
389406

390407
call_user_func($assertions, $response);

‎tests/resources/functions/node-long-coldstart/index.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
function fibo(n) {
2+
if (n < 2)
3+
return 1;
4+
else return fibo(n - 2) + fibo(n - 1);
5+
}
6+
7+
let cache = fibo(45);
8+
9+
module.exports = async (context) => {
10+
return context.res.send(cache);
11+
}

‎tests/resources/functions/node-long-coldstart/package.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "timeout",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "ISC"
12+
}

0 commit comments

Comments
 (0)