Skip to content

Commit c0e7943

Browse files
authored
KFLUXINFRA-2612 - Add Windows in host-config chart (#9505)
Add Windows in host-config chart and in staging
1 parent d9aca34 commit c0e7943

File tree

3 files changed

+220
-1
lines changed

3 files changed

+220
-1
lines changed

components/kueue/staging/stone-stage-p01/queue-config/cluster-queue.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ spec:
109109
- local
110110
- localhost
111111
- macos-mac2metal-arm64
112+
- windows-amd64
112113
flavors:
113114
- name: platform-group-2
114115
resources:
@@ -140,6 +141,8 @@ spec:
140141
nominalQuota: '1000'
141142
- name: macos-mac2metal-arm64
142143
nominalQuota: '5'
144+
- name: windows-amd64
145+
nominalQuota: '5'
143146
stopPolicy: None
144147
---
145148
apiVersion: kueue.x-k8s.io/v1beta1

components/multi-platform-controller/base/host-config-chart/templates/host-config.yaml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ data:
4040

4141
{{- $arm := (index .Values "archDefaults" "arm64") | default (dict) }}
4242
{{- $amd := (index .Values "archDefaults" "amd64") | default (dict) }}
43+
{{- $windows := (index .Values "archDefaults" "windows") | default (dict) }}
4344
{{- $environment := .Values.environment | default "prod" }}
4445

4546
# cpu:memory (1:4)
@@ -1117,6 +1118,38 @@ data:
11171118
{{- end }}
11181119
{{ end }}
11191120

1121+
# Windows platforms
1122+
{{- if hasKey .Values.dynamicConfigs "windows-amd64" }}
1123+
{{- $config := index .Values.dynamicConfigs "windows-amd64" | default (dict) }}
1124+
dynamic.windows-amd64.type: {{ index $config "type" | default "aws" | quote }}
1125+
dynamic.windows-amd64.region: {{ index $config "region" | default "us-east-1" | quote }}
1126+
dynamic.windows-amd64.ami: {{ default (index $windows "ami") $config.ami | quote }}
1127+
dynamic.windows-amd64.instance-type: {{ (index $config "instance-type") | default "c5.4xlarge" | quote }}
1128+
dynamic.windows-amd64.instance-tag: {{ (index $config "instance-tag") | default (printf "%s-amd64-windows" $environment) | quote }}
1129+
dynamic.windows-amd64.key-name: {{ default (index $windows "key-name") ((index $config "key-name")) | quote }}
1130+
dynamic.windows-amd64.aws-secret: {{ (index $config "aws-secret") | default "aws-account" | quote }}
1131+
dynamic.windows-amd64.ssh-secret: {{ (index $config "ssh-secret") | default "aws-ssh-key" | quote }}
1132+
dynamic.windows-amd64.security-group-id: {{ default (index $windows "security-group-id") ((index $config "security-group-id")) | quote }}
1133+
dynamic.windows-amd64.max-instances: {{ (index $config "max-instances") | default "5" | quote }}
1134+
dynamic.windows-amd64.subnet-id: {{ default (index $windows "subnet-id") ((index $config "subnet-id")) | quote }}
1135+
dynamic.windows-amd64.disk: {{ index $config "disk" | default "100" | quote }}
1136+
dynamic.windows-amd64.check-interval: {{ (index $config "check-interval") | default "60" | quote }}
1137+
dynamic.windows-amd64.allocation-timeout: "1200"
1138+
{{- if (index $config "iops") }}
1139+
dynamic.windows-amd64.iops: {{ index $config "iops" | quote }}
1140+
{{ end }}
1141+
{{- if (index $config "throughput") }}
1142+
dynamic.windows-amd64.throughput: {{ index $config "throughput" | quote }}
1143+
{{ end }}
1144+
{{- if (index $config "user-data") }}
1145+
dynamic.windows-amd64.user-data: |
1146+
{{- $lines := splitList "\n" (index $config "user-data") }}
1147+
{{- range $line := $lines }}
1148+
{{ $line }}
1149+
{{- end }}
1150+
{{- end }}
1151+
{{ end }}
1152+
11201153
# Static hosts configuration
11211154
{{- range $host, $config := .Values.staticHosts }}
11221155
{{- range $key, $value := $config }}

components/multi-platform-controller/staging-downstream/host-values.yaml

Lines changed: 184 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@ archDefaults:
2222
key-name: "konflux-stage-int-mab01"
2323
security-group-id: "sg-0482e8ccae008b240"
2424
subnet-id: "subnet-07597d1edafa2b9d3"
25-
25+
windows:
26+
ami: "ami-0cf643428c5013531"
27+
key-name: "konflux-stage-int-mab01"
28+
security-group-id: "sg-0482e8ccae008b240"
29+
subnet-id: "subnet-07597d1edafa2b9d3"
2630

2731
dynamicConfigs:
2832
linux-arm64:
@@ -232,6 +236,185 @@ dynamicConfigs:
232236
sudo-commands: "/usr/bin/podman"
233237
disk: "200"
234238

239+
windows-amd64:
240+
user-data: |
241+
<powershell>
242+
function Wait-Folder {
243+
param(
244+
[Parameter(Mandatory=$true)]
245+
[string]$FolderPath,
246+
247+
[Parameter(Mandatory=$false)]
248+
[int]$TimeoutSeconds = 30
249+
)
250+
Write-Host "Waiting for folder '${FolderPath}' to be created"
251+
252+
# Start a timer
253+
$stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
254+
255+
while (-not (Test-Path -Path ${FolderPath})) {
256+
# Check if we have exceeded the timeout
257+
if ($stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds) {
258+
Write-Error "Timeout reached! Folder was not created within $TimeoutSeconds seconds."
259+
$stopwatch.Stop()
260+
return $false
261+
}
262+
263+
Write-Host "Waiting for folder..." -NoNewline
264+
Start-Sleep -Seconds 1
265+
}
266+
$stopwatch.Stop()
267+
268+
return $true
269+
}
270+
271+
## -------------------------------------
272+
## --------- Create Local User ---------
273+
## -------------------------------------
274+
$user = "konflux-builder"
275+
if ((Get-LocalUser -Name "${user}" -ErrorAction SilentlyContinue) -eq $null) {
276+
$password = (-join([char[]](33..122) | Get-Random -Count 30))
277+
$securePassword = (ConvertTo-SecureString $password -AsPlainText -Force)
278+
279+
# Create user
280+
New-LocalUser -Name $user -Password $securePassword -Description "Konflux Builder" | Out-Null
281+
Add-LocalGroupMember -Group 'Users' -Member "${user}"
282+
Add-LocalGroupMember -Group 'OpenSSH Users' -Member "${user}"
283+
284+
# Create a Credential Object for the new user
285+
$userCred = New-Object System.Management.Automation.PSCredential($user, $securePassword)
286+
287+
# Start a dummy Process as the new User
288+
# This is required to have the user home folder initialized.
289+
# TODO: can we do better?
290+
Start-Process -FilePath "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
291+
-Credential ${userCred} `
292+
-ArgumentList "-Command exit" `
293+
-LoadUserProfile `
294+
-WindowStyle Hidden `
295+
-WorkingDirectory "C:\Users\" `
296+
-Wait
297+
298+
# Create a Key to login as user
299+
Write-Host "Creating SSH Key for user '${user}'"
300+
$tempKey = "${env:TEMP}\${user}"
301+
if (Test-Path "${tempKey}") { Remove-Item -Force "${tempKey}" }
302+
if (Test-Path "${tempKey}.pub") { Remove-Item -Force "${tempKey}.pub" }
303+
ssh-keygen -t rsa -f "${tempKey}" -N `"`" | Out-Null
304+
305+
# Move private key to a secure location and restrict access to it
306+
$privateKeyPath = "C:\Users\Administrator\${user}"
307+
mv "${env:TEMP}\${user}" "${privateKeyPath}"
308+
$ACL = Get-Acl "${privateKeyPath}"
309+
$Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("NT AUTHORITY\SYSTEM", "FullControl", "Allow")
310+
$ACL.SetAccessRule($Ar)
311+
$Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators", "FullControl", "Allow")
312+
$ACL.SetAccessRule($Ar)
313+
Set-Acl "${privateKeyPath}" ${ACL}
314+
315+
# Init home folder
316+
$userHome = "C:\Users\${user}"
317+
Write-Host "Waiting for User Home '${userHome}' to be created"
318+
319+
# Ensure User's home folder is eventually created
320+
if (-not (Wait-Folder -FolderPath ${userHome})) {
321+
Write-Error "Folder '${userHome}' not found! Cleanup..." -ForegroundColor Red
322+
if (Test-Path "\${tempKey}") { Remove-Item -Force "${tempKey}" }
323+
if (Test-Path "\${tempKey}.pub") { Remove-Item -Force "${tempKey}.pub" }
324+
exit 1
325+
}
326+
327+
# Set-up SSH Keys for User
328+
Write-Host "User Home found. Configuring SSH access" -ForegroundColor Green
329+
New-Item -ItemType Directory -Force -Path "${userHome}\.ssh"
330+
New-Item -ItemType Directory -Force -Path "${userHome}\build"
331+
332+
# Copying and removing to preserve file permissions! Do not use `mv`! :)
333+
cp "${tempKey}.pub" "${userHome}\.ssh\authorized_keys"
334+
rm "${tempKey}.pub"
335+
}
336+
337+
## ---------------------------------------------
338+
## --------- Enable Windows Containers ---------
339+
## ---------------------------------------------
340+
Invoke-WebRequest -UseBasicParsing "https://raw.githubusercontent.com/microsoft/Windows-Containers/Main/helpful_tools/Install-DockerCE/install-docker-ce.ps1" -o install-docker-ce.ps1
341+
.\install-docker-ce.ps1 -NoRestart
342+
if (${global:RebootRequired}) {
343+
Restart-Computer
344+
exit
345+
}
346+
347+
# Create docker-users group and add konflux-builder to it
348+
if ((Get-LocalGroup -Name 'docker-users') -eq $null) {
349+
New-LocalGroup -Name 'docker-users' -Description 'Docker Users'
350+
}
351+
if ((Get-LocalGroupMember -Group 'docker-users' -Member 'konflux-builder') -eq $null) {
352+
Add-LocalGroupMember -Group 'docker-users' -Member "${user}"
353+
}
354+
355+
# allow the docker-users group to use docker
356+
$dockerConfigPath = "C:\ProgramData\docker\config\daemon.json"
357+
$existingConfig = Get-Content $dockerConfigPath -Raw | ConvertFrom-Json
358+
if ((${existingConfig}.group) -eq $null) {
359+
$existingConfig | Add-Member -NotePropertyName "group" -NotePropertyValue "docker-users" -Force
360+
$existingConfig | ConvertTo-Json -Depth 10 | Set-Content $dockerConfigPath
361+
Restart-Service docker
362+
}
363+
364+
# Exclude docker in Windows Defender
365+
Add-MpPreference -ExclusionProcess "dockerd.exe"
366+
Add-MpPreference -ExclusionProcess "docker.exe"
367+
Add-MpPreference -ExclusionProcess "containerd.exe"
368+
Add-MpPreference -ExclusionProcess "vmcompute.exe"
369+
370+
## -------------------------------------
371+
## --------- Configure OpenSSH ---------
372+
## -------------------------------------
373+
374+
# Install OpenSSH Server
375+
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
376+
377+
# Start the sshd service and set it to start automatically
378+
Start-Service sshd
379+
Set-Service -Name sshd -StartupType 'Automatic'
380+
381+
# Grab the Public Key from AWS Metadata and configure authorized_keys
382+
# This allows you to log in with your .pem/.ppk file instead of a password
383+
$MAGIC_IP = "169.254.169.254"
384+
$IMDS_TOKEN = Invoke-RestMethod -Uri "http://${MAGIC_IP}/latest/api/token" -Method 'PUT' -Headers @{'X-aws-ec2-metadata-token-ttl-seconds' = '21600'}
385+
$PUBKEY = Invoke-RestMethod -Uri "http://${MAGIC_IP}/latest/meta-data/public-keys/0/openssh-key" -Headers @{'X-aws-ec2-metadata-token' = $IMDS_TOKEN}
386+
387+
# Ensure SSH_PATH folder was created
388+
$SSH_PATH = "C:\ProgramData\ssh"
389+
Write-Host "Waiting for SSH Folder"
390+
if (-not (Wait-Folder -FolderPath ${SSH_PATH})) {
391+
Write-Error "Folder '${SSH_PATH}' not found! Exiting..." -ForegroundColor Red
392+
exit 1
393+
}
394+
Write-Host "Folder '${SSH_PATH}' found"
395+
396+
# Add key to administrators_authorized_keys
397+
$PUBKEY | Out-File -FilePath "$SSH_PATH\administrators_authorized_keys" -Encoding ascii
398+
399+
# Fix permissions (ACLs) for the authorized_keys file
400+
# OpenSSH is strict: only System and Administrators should have access
401+
$ACL = Get-Acl "$SSH_PATH\administrators_authorized_keys"
402+
$Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("NT AUTHORITY\SYSTEM", "FullControl", "Allow")
403+
$ACL.SetAccessRule($Ar)
404+
$Ar = New-Object System.Security.AccessControl.FileSystemAccessRule("BUILTIN\Administrators", "FullControl", "Allow")
405+
$ACL.SetAccessRule($Ar)
406+
Set-Acl "$SSH_PATH\administrators_authorized_keys" $ACL
407+
408+
# Restart sshd to apply key changes
409+
Restart-Service sshd
410+
411+
# Configure the Firewall to allow SSH (Port 22)
412+
Remove-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' | Out-Null
413+
New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -RemoteAddress any
414+
Write-Output "Firewall Rule 'OpenSSH-Server-In-TCP' updated"
415+
</powershell>
416+
<persist>true</persist>
417+
235418
# Static hosts configuration
236419
staticHosts:
237420
ppc64le-static-1:

0 commit comments

Comments
 (0)