Skip to content

Commit 6860e4a

Browse files
timfeltimfel
committed
[GR-21590][GR-68661] Update imports and tighten up PGO profile selection
PullRequest: graalpython/3969
2 parents ff75877 + 06102f1 commit 6860e4a

File tree

19 files changed

+329
-221
lines changed

19 files changed

+329
-221
lines changed

‎.github/workflows/platforms-tests.yml‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ jobs:
2323
mx deploy-local-maven-repo
2424
mv mxbuild/jdk*/mx.graalpython/public-maven-repo m2repo
2525
26-
- name: Test on OpenJ9
26+
- name: Install OpenJ9 17
27+
uses: actions/setup-java@v4
28+
with:
29+
distribution: 'semeru'
30+
java-version: '17'
31+
- name: Test on OpenJ9 17
2732
run: |
28-
# Install IBM Semeru OpenJ9 Java 21
29-
wget https://github.com/ibmruntimes/semeru21-binaries/releases/download/jdk-21.0.8%2B9_openj9-0.53.0/ibm-semeru-open-jdk_x64_linux_21.0.8_9_openj9-0.53.0.tar.gz
30-
mkdir -p /opt/java/openjdk-21-openj9
31-
tar -C /opt/java/openjdk-21-openj9 --strip-components=1 -xzf ibm-semeru*.tar.gz
32-
export JAVA_HOME=/opt/java/openjdk-21-openj9
33-
34-
export PATH=$JAVA_HOME/bin:$PATH
33+
echo "Using $JAVA_HOME"
34+
$JAVA_HOME/bin/java -version
3535
mkdir $(pwd)/user_resource_cache
3636
mvn -f graalpython/com.oracle.graal.python.test.integration/pom.xml -Dcom.oracle.graal.python.test.polyglot.version=26.0.0 -Dcom.oracle.graal.python.test.polyglot_repo=file:///$(pwd)/m2repo --batch-mode -U -Dtruffle.UseFallbackRuntime=true -Dpolyglot.engine.allowUnsupportedPlatform=true -Dpolyglot.engine.userResourceCache=/$(pwd)/user_resource_cache -Dpolyglot.python.UnsupportedPlatformEmulates=linux -Dorg.graalvm.python.resources.exclude=native.files test -Dtest=HelloWorldTests,AttributeTests,BuiltinSubclassTest,ComplexTexts,CreateClassTest,AsyncActionThreadingTest,JavaInteropTest
3737
rm -rf $(pwd)/user_resource_cache
@@ -46,15 +46,12 @@ jobs:
4646
--memory-reservation 6G
4747
run: |
4848
apt-get update
49-
apt-get install -y python3 git wget tar gzip cmake build-essential maven openjdk-21-jdk
50-
51-
export JAVA_HOME=/usr/lib/jvm/java-21-openjdk-ppc64el
49+
apt-get install -y python3 git wget tar gzip cmake build-essential maven openjdk-17-jdk
50+
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-ppc64el
5251
export PATH=$JAVA_HOME/bin:$PATH
5352
5453
echo "Using $JAVA_HOME"
5554
$JAVA_HOME/bin/java -version
56-
57-
# Run some basic smoke tests
5855
cd /workspace
5956
mkdir $(pwd)/user_resource_cache
6057
mvn -f graalpython/com.oracle.graal.python.test.integration/pom.xml -Dcom.oracle.graal.python.test.polyglot.version=26.0.0 -Dcom.oracle.graal.python.test.polyglot_repo=file:///$(pwd)/m2repo --batch-mode -U -Dtruffle.UseFallbackRuntime=true -Dpolyglot.engine.allowUnsupportedPlatform=true -Dpolyglot.engine.userResourceCache=/$(pwd)/user_resource_cache -Dpolyglot.python.UnsupportedPlatformEmulates=linux -Dorg.graalvm.python.resources.exclude=native.files test -Dtest=HelloWorldTests,AttributeTests,BuiltinSubclassTest,ComplexTexts,CreateClassTest,AsyncActionThreadingTest,JavaInteropTest

‎ci.jsonnet‎

Lines changed: 7 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -88,36 +88,6 @@
8888
name_suffix +:: ["bytecode-dsl"],
8989
}),
9090

91-
// -----------------------------------------------------------------------------------------------------------------
92-
//
93-
// extra wheel builder configs.
94-
//
95-
// these are definitions to build wheels for releases which aren't build as
96-
// part of the normal (MCD) wheel builder infrastructure. the urls to the
97-
// release binaries and which wheels to build are just updated manually
98-
// here as needed. Wheels in particular simply have to be tested (at least
99-
// superficially) by someone and then be added to the list.
100-
//
101-
// -----------------------------------------------------------------------------------------------------------------
102-
local WHEELS_COMMON = [
103-
"numpy",
104-
"httptools",
105-
"kiwisolver",
106-
"psutil",
107-
"ujson",
108-
"xxhash",
109-
],
110-
local GPY_RELEASE = {
111-
windows: {
112-
url: $.overlay_imports.RELEASES_BASE_URL + "python-native-standalone-svm-svmee-java23-windows-amd64/24.1.1/python-native-standalone-svm-svmee-java23-windows-amd64-24.1.1-signed.zip",
113-
wheels: WHEELS_COMMON,
114-
},
115-
darwin: {
116-
url: $.overlay_imports.RELEASES_BASE_URL + "python-native-standalone-svm-svmee-java23-darwin-aarch64/24.1.1/python-native-standalone-svm-svmee-java23-darwin-aarch64-24.1.1-signed.tar.gz",
117-
wheels: WHEELS_COMMON,
118-
},
119-
},
120-
12191
// -----------------------------------------------------------------------------------------------------------------
12292
//
12393
// main build definition (matrix)
@@ -229,9 +199,12 @@
229199
}),
230200
"python-svm-build": gpgate + platform_spec(no_jobs) + platform_spec({
231201
"linux:amd64:jdk-latest" : tier2 + provide(GPY_NATIVE_STANDALONE),
202+
"linux:aarch64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
203+
"darwin:aarch64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
204+
"windows:amd64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
232205
}),
233206
"python-pgo-profile": gpgate_ee + platform_spec(no_jobs) + platform_spec({
234-
"linux:amd64:jdk-latest" : weekly + t("01:30:00") + task_spec({
207+
"linux:amd64:jdk-latest" : post_merge + t("01:30:00") + task_spec({
235208
run: [["mx", "python-native-pgo"]],
236209
logs+: [
237210
"default.iprof.gz",
@@ -241,10 +214,10 @@
241214
}),
242215
"python-svm-unittest": gpgate + platform_spec(no_jobs) + platform_spec({
243216
"linux:amd64:jdk-latest" : tier2 + require(GPY_NATIVE_STANDALONE),
244-
"linux:aarch64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
217+
"linux:aarch64:jdk-latest" : tier3 + require(GPY_NATIVE_STANDALONE),
245218
"darwin:amd64:jdk-latest" : daily + t("01:30:00") + provide(GPY_NATIVE_STANDALONE),
246-
"darwin:aarch64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
247-
"windows:amd64:jdk-latest" : tier3 + provide(GPY_NATIVE_STANDALONE),
219+
"darwin:aarch64:jdk-latest" : tier3 + require(GPY_NATIVE_STANDALONE),
220+
"windows:amd64:jdk-latest" : tier3 + require(GPY_NATIVE_STANDALONE) + batches(2),
248221
}),
249222
"python-svm-unittest-bytecode-dsl": gpgate + platform_spec(no_jobs) + bytecode_dsl_gate("python-svm-unittest") + platform_spec({
250223
"linux:amd64:jdk-latest" : tier2 + provide(GPY_NATIVE_BYTECODE_DSL_STANDALONE),
@@ -314,25 +287,6 @@
314287
"tox-example": gpgate_ee + require(GPYEE_NATIVE_STANDALONE) + platform_spec(no_jobs) + platform_spec({
315288
"linux:amd64:jdk-latest" : tier3,
316289
}),
317-
"build-wheels": base_gate + platform_spec(no_jobs) + platform_spec({
318-
"windows:amd64:jdk-latest" : on_demand + t("01:00:00"),
319-
"darwin:aarch64:jdk-latest" : on_demand + t("01:00:00"),
320-
}) + task_spec({
321-
setup: [
322-
["python", "-m", "venv", ".venv"],
323-
["set-export", "PACKAGES_TO_BUILD", std.join(",", GPY_RELEASE[self.os].wheels)],
324-
],
325-
run: [
326-
[
327-
if self.os == "windows" then ".venv/Scripts/python.exe" else ".venv/bin/python",
328-
"scripts/wheelbuilder/build_wheels.py",
329-
GPY_RELEASE[self.os].url,
330-
]
331-
],
332-
logs+: [
333-
"wheelhouse/*.whl",
334-
],
335-
}),
336290
},
337291

338292
local forks_warmup = forks("./mx.graalpython/warmup-fork-counts.json"),

‎ci/graal/ci/ci_common/common.jsonnet‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,8 @@ common + common.frequencies + {
138138
linux_amd64: common.linux_amd64 + graal_common_extras + linux_deps_extras,
139139
linux_amd64_ol9: common.linux_amd64_ol9 + graal_common_extras + linux_deps_extras,
140140
linux_amd64_ubuntu: common.linux_amd64_ubuntu + graal_common_extras,
141-
linux_aarch64: linux_deps_extras + common.linux_aarch64 + graal_common_extras,
142-
linux_aarch64_ol9: linux_deps_extras + common.linux_aarch64_ol9 + graal_common_extras,
141+
linux_aarch64: common.linux_aarch64 + graal_common_extras + linux_deps_extras,
142+
linux_aarch64_ol9: common.linux_aarch64_ol9 + graal_common_extras + linux_deps_extras,
143143
darwin_amd64: common.darwin_amd64 + graal_common_extras,
144144
darwin_aarch64: common.darwin_aarch64 + graal_common_extras,
145145
windows_amd64: common.windows_amd64 + graal_common_extras,

‎ci/graal/ci/common.jsonnet‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ local common_json = import "../common.json";
6565
for name in ["oraclejdk-latest"] + variants("labsjdk-ce-latest") + variants("labsjdk-ee-latest")
6666
} + {
6767
'graalvm-ee-21': jdk_base + common_json.jdks["graalvm-ee-21"] + { jdk_version:: 21 },
68+
'graalvm-ee-25-ea': jdk_base + common_json.jdks["graalvm-ee-25-ea"] + { jdk_version:: 25 },
6869
},
6970
# We do not want to expose galahad-jdk
7071
assert std.assertEqual([x for x in std.objectFields(common_json.jdks) if x != "galahad-jdk"], std.objectFields(jdks_data)),
@@ -109,7 +110,8 @@ local common_json = import "../common.json";
109110
oraclejdkLatest: self["oraclejdk-latest"],
110111
},
111112

112-
# The devkits versions reflect those used to build the JVMCI JDKs (e.g., see devkit_platform_revisions in <jdk>/make/conf/jib-profiles.js)
113+
# The devkits versions reflect those used to build the JVMCI JDKs (e.g., see devkit_platform_revisions in <jdk>/make/conf/jib-profiles.js).
114+
# See deps.windows_devkit to add a devkit on windows conveniently.
113115
devkits: {
114116
"windows-jdk17": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }},
115117
"windows-jdk19": { packages+: { "devkit:VS2022-17.1.0+1": "==0" }},
@@ -174,6 +176,12 @@ local common_json = import "../common.json";
174176

175177
# These dependencies are not included by default in any platform object
176178

179+
# Not included by default in $.windows_amd64 and $.windows_server_2016_amd64 because it needs jdk_name.
180+
# As a note, Native Image needs this to build.
181+
windows_devkit:: {
182+
packages+: if self.os == "windows" then $.devkits["windows-" + self.jdk_name].packages else {},
183+
},
184+
177185
eclipse: {
178186
downloads+: {
179187
ECLIPSE: {
@@ -239,6 +247,29 @@ local common_json = import "../common.json";
239247
} else {},
240248
},
241249

250+
maven:: {
251+
local this = self,
252+
packages+: (if self.os == "linux" && self.arch == "amd64" then {
253+
maven: '==3.9.10',
254+
} else {}),
255+
# no maven package available on other platforms
256+
downloads+: (if self.os != "linux" || self.arch != "amd64" then {
257+
# GR-68921: 3.9.10 does not work on Windows
258+
MAVEN_HOME: {name: 'maven', version: (if this.os == "windows" then '3.3.9' else '3.9.10'), platformspecific: false},
259+
} else {}),
260+
setup+: (if self.os != "linux" || self.arch != "amd64" then [
261+
['set-export', 'PATH', (if self.os == "windows" then '$MAVEN_HOME\\bin;$PATH' else '$MAVEN_HOME/bin:$PATH')],
262+
] else []),
263+
},
264+
265+
espresso:: {
266+
downloads+: {
267+
EXTRA_JAVA_HOMES+: {
268+
pathlist+: [jdks_data["oraclejdk21"], jdks_data["oraclejdk25"]],
269+
}
270+
}
271+
},
272+
242273
sulong:: self.cmake + {
243274
packages+: if self.os == "windows" then {
244275
msvc_source: "==14.0",
@@ -275,12 +306,11 @@ local common_json = import "../common.json";
275306
} else {},
276307
},
277308

278-
graalpy:: self.gradle + self.cmake + {
309+
graalpy:: self.gradle + self.cmake + self.maven + {
279310
packages+: if (self.os == "linux") then {
280311
libffi: '==3.2.1',
281312
bzip2: '==1.0.6',
282313
zlib: '==1.2.11',
283-
maven: "==3.6.3",
284314
} else {},
285315
},
286316

‎ci/graal/common.json‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44
"Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
55
],
66

7-
"mx_version": "7.58.9",
7+
"mx_version": "7.59.0",
88

99
"COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
1010
"jdks": {
11-
"galahad-jdk": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+10-1012", "platformspecific": true, "extrabundles": ["static-libs"]},
11+
"galahad-jdk": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+11-1117", "platformspecific": true, "extrabundles": ["static-libs"]},
1212

1313
"oraclejdk17": {"name": "jpg-jdk", "version": "17.0.7", "build_id": "jdk-17.0.7+8", "platformspecific": true, "extrabundles": ["static-libs"]},
1414
"labsjdk-ce-17": {"name": "labsjdk", "version": "ce-17.0.7+4-jvmci-23.1-b02", "platformspecific": true },
@@ -52,14 +52,15 @@
5252
"labsjdk-ee-25": {"name": "labsjdk", "version": "ee-25+30-jvmci-b01", "platformspecific": true },
5353
"labsjdk-ee-25Debug": {"name": "labsjdk", "version": "ee-25+30-jvmci-b01-debug", "platformspecific": true },
5454
"labsjdk-ee-25-llvm": {"name": "labsjdk", "version": "ee-25+30-jvmci-b01-sulong", "platformspecific": true },
55+
"graalvm-ee-25-ea": {"name": "graalvm-jdk", "version": "25.0.0", "ea": "33", "platformspecific": true },
5556

56-
"oraclejdk-latest": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+10", "platformspecific": true, "extrabundles": ["static-libs"]},
57-
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-26+10-jvmci-b01", "platformspecific": true },
58-
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-26+10-jvmci-b01-debug", "platformspecific": true },
59-
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-26+10-jvmci-b01-sulong", "platformspecific": true },
60-
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-26+10-jvmci-b01", "platformspecific": true },
61-
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-26+10-jvmci-b01-debug", "platformspecific": true },
62-
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-26+10-jvmci-b01-sulong", "platformspecific": true }
57+
"oraclejdk-latest": {"name": "jpg-jdk", "version": "26", "build_id": "jdk-26+11", "platformspecific": true, "extrabundles": ["static-libs"]},
58+
"labsjdk-ce-latest": {"name": "labsjdk", "version": "ce-26+11-jvmci-b01", "platformspecific": true },
59+
"labsjdk-ce-latestDebug": {"name": "labsjdk", "version": "ce-26+11-jvmci-b01-debug", "platformspecific": true },
60+
"labsjdk-ce-latest-llvm": {"name": "labsjdk", "version": "ce-26+11-jvmci-b01-sulong", "platformspecific": true },
61+
"labsjdk-ee-latest": {"name": "labsjdk", "version": "ee-26+11-jvmci-b01", "platformspecific": true },
62+
"labsjdk-ee-latestDebug": {"name": "labsjdk", "version": "ee-26+11-jvmci-b01-debug", "platformspecific": true },
63+
"labsjdk-ee-latest-llvm": {"name": "labsjdk", "version": "ee-26+11-jvmci-b01-sulong", "platformspecific": true }
6364
},
6465

6566
"eclipse": {

‎ci/python-bench.libsonnet‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,7 @@
147147
vm:: [],
148148

149149
// public
150+
deploysArtifacts: true,
150151
capabilities+: capabilities(self.os, self.arch),
151152
downloads: downloads(self.os, self.arch),
152153
targets: [self.target] + if self.target == 'bench' then [] else ['bench'],

‎ci/python-gate.libsonnet‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,7 @@
423423
"--strict-compliance", "--primary", "gate", "--tags", self.tags, "-B=--force-deprecation-as-warning",
424424
] + self.all_suites + self.gate_parameters,
425425
],
426+
deploysArtifacts: true,
426427
}),
427428

428429
cpython_gate:: base_gate + test_reports + task_spec({

‎graalpython/com.oracle.graal.python.benchmarks/python/micro/magic-iter.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,19 @@
3838
# SOFTWARE.
3939

4040
class CustomIterator:
41+
# Even though this is worse for our generated code, it is increasingly
42+
# common, because the typing PEP explicitly has an example like this:
43+
# https://peps.python.org/pep-0526/#class-and-instance-variable-annotations
44+
#
45+
# While that is a historical document, it is a) still widely referenced and
46+
# b) at the time of writing this comment latest spec still has a similar
47+
# example where a variable is initialized with a default on the class level
48+
# and the __init__ method only conditionally overrides the default *and
49+
# otherwise keeps it on the class* making self.FOO accesses polymorphic:
50+
# https://typing.python.org/en/latest/spec/class-compat.html#classvar
51+
pos = 0
52+
4153
def __init__(self, obj):
42-
self.pos = 0
4354
self.__obj = obj
4455

4556
def __iter__(self):
@@ -70,7 +81,7 @@ def count(num):
7081
val0 = next(it)
7182
val1 = next(it)
7283
val2 = next(it)
73-
84+
7485
return (val0, val1, val2)
7586

7687

‎graalpython/com.oracle.graal.python.test/src/tests/test_socket.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -69,6 +69,12 @@ def test_get_name_info():
6969

7070

7171
def test_recv_into():
72+
try:
73+
if __graalpython__.posix_module_backend() == "java":
74+
return # transient timeouts
75+
except NameError:
76+
pass # not graalpy
77+
7278
port = None
7379
event = threading.Event()
7480
def server():

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_capi.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ test.test_capi.test_long.LongTests.test_compact @ darwin-arm64,linux-aarch64,lin
168168
test.test_capi.test_long.LongTests.test_long_aslongandoverflow @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
169169
test.test_capi.test_long.LongTests.test_long_aslonglong @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
170170
test.test_capi.test_long.LongTests.test_long_aslonglongandoverflow @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
171-
test.test_capi.test_long.LongTests.test_long_aspid @ linux-x86_64
171+
# Transiently fails with OverflowError: Python int too large to convert to C int
172+
!test.test_capi.test_long.LongTests.test_long_aspid @ linux-x86_64
172173
test.test_capi.test_long.LongTests.test_long_asunsignedlong @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
173174
test.test_capi.test_long.LongTests.test_long_asunsignedlonglong @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64
174175
test.test_capi.test_long.LongTests.test_long_asunsignedlonglongmask @ darwin-arm64,darwin-x86_64,linux-aarch64,linux-x86_64,win32-AMD64

0 commit comments

Comments
 (0)