fix: batch job processing

.github/workflows/forky-release.yml

@@ -0,0 +1,58 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      releaseVersion:
+        description: 'Version to release'
+        required: true
+jobs:
+  main:
+    # We want to keep this repository an official fork
+    # So this needs to stay public...
+    # Target only public infra (cloud runners, github packages, etc)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 21
+          distribution: adopt
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "22.x"
+
+      - name: Install node modules
+        run: npm ci
+
+      - name: Run get new version
+        run: npm run release-dry
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: BootJar with version
+        run: ./gradlew bootJar
+        env:
+          VERSION: ${{ github.event.inputs.releaseVersion }}
+
+      - name: Prepare for docker build
+        run: ./gradlew dockerPrepare
+        env:
+          VERSION: ${{ github.event.inputs.releaseVersion }}
+
+      - name: Create docker image
+        env:
+          DOCKER_REGISTRY: "ghcr.io"
+          IMAGE_PATH: "rakutenfrance/tolgee"
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.DOCKER_REGISTRY }} -u $ --password-stdin
+          docker build . -t ${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_PATH }}:${{ github.event.inputs.releaseVersion }} --platform linux/amd64
+          docker push ${{ env.DOCKER_REGISTRY }}/${{ env.IMAGE_PATH }}:${{ github.event.inputs.releaseVersion }}
+        working-directory: build/docker
+
+      - name: Pack with webapp
+        run: ./gradlew packResources
+        env:
+          VERSION: ${{ github.event.inputs.releaseVersion }}

backend/api/src/main/kotlin/io/tolgee/api/v2/controllers/ProjectBatchLockController.kt

@@ -0,0 +1,150 @@
+package io.tolgee.api.v2.controllers
+
+import io.swagger.v3.oas.annotations.Operation
+import io.swagger.v3.oas.annotations.tags.Tag
+import io.tolgee.batch.BatchJobChunkExecutionQueue
+import io.tolgee.batch.BatchJobProjectLockingManager
+import io.tolgee.batch.BatchJobService
+import io.tolgee.batch.JobCharacter
+import io.tolgee.model.batch.BatchJobStatus
+import io.tolgee.openApiDocs.OpenApiSelfHostedExtension
+import io.tolgee.security.authentication.RequiresSuperAuthentication
+import io.tolgee.util.Logging
+import io.tolgee.util.logger
+import org.springframework.hateoas.CollectionModel
+import org.springframework.web.bind.annotation.CrossOrigin
+import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+
+/**
+ * REST API for managing project batch job locks
+ */
+@RestController
+@CrossOrigin(origins = ["*"])
+@RequestMapping("/v2/administration")
+@Tag(
+  name = "Server Administration",
+  description = "**Only for self-hosted instances** \n\n" +
+    "Management of project-level batch job locks and queue inspection for debugging and maintenance."
+)
+@OpenApiSelfHostedExtension
+class ProjectBatchLockController(
+  private val batchJobProjectLockingManager: BatchJobProjectLockingManager,
+  private val batchJobService: BatchJobService,
+  private val batchJobChunkExecutionQueue: BatchJobChunkExecutionQueue,
+) : IController, Logging {
+
+  @GetMapping("/project-batch-locks")
+  @Operation(
+    summary = "Get all project batch locks",
+    description = "Returns current project batch job locks from Redis or local storage based on configuration"
+  )
+  @RequiresSuperAuthentication
+  fun getProjectLocks(): CollectionModel<ProjectLockModel> {
+    logger.debug("Retrieving all project batch locks")
+
+    val locks = batchJobProjectLockingManager.getMap()
+    val lockModels = locks.entries.map { (projectId, lockedJobIds) ->
+      createProjectLockModel(projectId, lockedJobIds)
+    }
+
+    logger.debug("Retrieved ${lockModels.size} project batch locks")
+    return CollectionModel.of(lockModels)
+  }
+
+  private fun createProjectLockModel(projectId: Long, lockedJobIds: Set<Long>): ProjectLockModel {
+    val lockStatus = when {
+      lockedJobIds.isEmpty() -> LockStatus.UNLOCKED
+      else -> LockStatus.LOCKED
+    }
+
+    val jobInfos = lockedJobIds.mapNotNull { jobId ->
+      try {
+        val jobDto = batchJobService.getJobDto(jobId)
+        JobInfo(
+          jobId = jobDto.id,
+          status = jobDto.status,
+          type = jobDto.type,
+          createdAt = jobDto.createdAt
+        )
+      } catch (e: Exception) {
+        logger.warn("Could not retrieve job info for locked job $jobId in project $projectId", e)
+        null
+      }
+    }
+
+    return ProjectLockModel(
+      projectId = projectId,
+      lockedJobIds = lockedJobIds,
+      lockStatus = lockStatus,
+      jobInfos = jobInfos
+    )
+  }
+
+  @GetMapping("/batch-job-queue")
+  @Operation(
+    summary = "Get current batch job queue",
+    description = "Returns all chunk execution items currently in the batch job queue"
+  )
+  @RequiresSuperAuthentication
+  fun getBatchJobQueue(): CollectionModel<QueueItemModel> {
+    logger.debug("Retrieving current batch job queue")
+
+    val queueItems = batchJobChunkExecutionQueue.getAllQueueItems()
+    val queueModels = queueItems.map { item ->
+      QueueItemModel(
+        chunkExecutionId = item.chunkExecutionId,
+        jobId = item.jobId,
+        executeAfter = item.executeAfter,
+        jobCharacter = item.jobCharacter,
+        managementErrorRetrials = item.managementErrorRetrials
+      )
+    }
+
+    logger.debug("Retrieved ${queueModels.size} items from batch job queue")
+    return CollectionModel.of(queueModels)
+  }
+}
+
+/**
+ * Model representing a project batch lock
+ */
+data class ProjectLockModel(
+  val projectId: Long,
+  val lockedJobIds: Set<Long>,
+  val lockStatus: LockStatus,
+  val jobInfos: List<JobInfo>
+)
+
+/**
+ * Information about the locked job
+ */
+data class JobInfo(
+  val jobId: Long,
+  val status: BatchJobStatus,
+  val type: io.tolgee.batch.data.BatchJobType,
+  val createdAt: Long?
+)
+
+/**
+ * Status of the project lock
+ */
+enum class LockStatus {
+  /** Project lock is explicitly cleared (value = empty set) */
+  UNLOCKED,
+
+  /** Project lock is held by one or more jobs (value = set of job IDs) */
+  LOCKED
+}
+
+/**
+ * Model representing a queue item for batch job chunk execution
+ */
+data class QueueItemModel(
+  val chunkExecutionId: Long,
+  val jobId: Long,
+  val executeAfter: Long?,
+  val jobCharacter: JobCharacter,
+  val managementErrorRetrials: Int
+)

backend/app/src/main/resources/logback-spring.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+    <springProfile name="!json-logging">
+        <!-- Default console appender for non-JSON logging -->
+        <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+            <encoder>
+                <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            </encoder>
+        </appender>
+
+        <root level="INFO">
+            <appender-ref ref="CONSOLE"/>
+        </root>
+    </springProfile>
+
+    <springProfile name="json-logging">
+        <!-- JSON console appender using standard logback pattern -->
+        <appender name="JSON_CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+            <encoder>
+                <pattern>{"timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ}","level":"%p","logger":"%c","message":"%m","thread":"%t","exception":"%ex{full}"}%n</pattern>
+            </encoder>
+        </appender>
+
+        <root level="INFO">
+            <appender-ref ref="JSON_CONSOLE"/>
+        </root>
+    </springProfile>
+</configuration>

backend/app/src/test/kotlin/io/tolgee/api/v2/controllers/ProjectBatchLockControllerTest.kt

@@ -0,0 +1,135 @@
+package io.tolgee.api.v2.controllers
+
+import io.tolgee.batch.BatchJobChunkExecutionQueue
+import io.tolgee.batch.BatchJobProjectLockingManager
+import io.tolgee.batch.BatchJobService
+import io.tolgee.batch.data.ExecutionQueueItem
+import io.tolgee.batch.data.BatchJobDto
+import io.tolgee.batch.data.BatchJobType
+import io.tolgee.development.testDataBuilder.data.AdministrationTestData
+import io.tolgee.fixtures.andIsOk
+import io.tolgee.fixtures.andIsUnauthorized
+import io.tolgee.model.batch.BatchJobStatus
+import io.tolgee.testing.AuthorizedControllerTest
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.mockito.kotlin.whenever
+import org.springframework.boot.test.mock.mockito.MockBean
+import java.util.concurrent.ConcurrentHashMap
+
+class ProjectBatchLockControllerTest : AuthorizedControllerTest() {
+
+  @MockBean
+  private lateinit var batchJobProjectLockingManager: BatchJobProjectLockingManager
+
+  @MockBean
+  private lateinit var batchJobService: BatchJobService
+
+  @MockBean
+  private lateinit var batchJobChunkExecutionQueue: BatchJobChunkExecutionQueue
+
+  lateinit var testData: AdministrationTestData
+
+  @BeforeEach
+  fun createData() {
+    testData = AdministrationTestData()
+    testDataService.saveTestData(testData.root)
+    userAccount = testData.admin
+  }
+
+  @Test
+  fun `GET project-batch-locks returns unauthorized without super auth`() {
+    // Test without admin user
+    userAccount = testData.user
+    performAuthGet("/v2/administration/project-batch-locks")
+      .andIsUnauthorized
+  }
+
+  @Test
+  fun `GET project-batch-locks returns locks with super auth`() {
+    val testLocks = ConcurrentHashMap<Long, Set<Long>>().apply {
+      put(1L, setOf(123L)) // Project 1 locked to job 123
+      put(2L, emptySet()) // Project 2 explicitly unlocked
+      put(3L, emptySet()) // Project 3 no jobs locked
+    }
+
+    whenever(batchJobProjectLockingManager.getMap()).thenReturn(testLocks)
+
+    // Mock job info for locked job
+    val mockJobDto = BatchJobDto(
+      id = 123L,
+      projectId = 1L,
+      authorId = 1L,
+      target = emptyList(),
+      totalItems = 100,
+      totalChunks = 10,
+      chunkSize = 10,
+      status = BatchJobStatus.RUNNING,
+      type = BatchJobType.MACHINE_TRANSLATE,
+      params = null,
+      maxPerJobConcurrency = 1,
+      jobCharacter = io.tolgee.batch.JobCharacter.FAST,
+      hidden = false,
+      debouncingKey = null,
+      createdAt = System.currentTimeMillis()
+    )
+
+    whenever(batchJobService.getJobDto(123L)).thenReturn(mockJobDto)
+
+    performAuthGet("/v2/administration/project-batch-locks")
+      .andIsOk
+  }
+
+  @Test
+  fun `PUT clear project lock works with super auth`() {
+    val testLocks = ConcurrentHashMap<Long, Set<Long>>()
+    whenever(batchJobProjectLockingManager.getMap()).thenReturn(testLocks)
+
+    performAuthPut("/v2/administration/project-batch-locks/123/clear", null)
+      .andIsOk
+  }
+
+  @Test
+  fun `DELETE project lock works with super auth`() {
+    val testLocks = ConcurrentHashMap<Long, Set<Long>>().apply {
+      put(123L, setOf(456L))
+    }
+    whenever(batchJobProjectLockingManager.getMap()).thenReturn(testLocks)
+
+    performAuthDelete("/v2/administration/project-batch-locks/123")
+      .andIsOk
+  }
+
+  @Test
+  fun `GET batch-job-queue returns queue items with super auth`() {
+    val queueItems = listOf(
+      ExecutionQueueItem(
+        chunkExecutionId = 1001L,
+        jobId = 2001L,
+        executeAfter = System.currentTimeMillis(),
+        jobCharacter = io.tolgee.batch.JobCharacter.FAST,
+        managementErrorRetrials = 0
+      ),
+      ExecutionQueueItem(
+        chunkExecutionId = 1002L,
+        jobId = 2002L,
+        executeAfter = null,
+        jobCharacter = io.tolgee.batch.JobCharacter.SLOW,
+        managementErrorRetrials = 1
+      )
+    )
+
+    whenever(batchJobChunkExecutionQueue.getAllQueueItems()).thenReturn(queueItems)
+
+    performAuthGet("/v2/administration/batch-job-queue")
+      .andIsOk
+  }
+
+  @Test
+  fun `GET batch-job-queue returns unauthorized without super auth`() {
+    // Test without admin user
+    userAccount = testData.user
+    performAuthGet("/v2/administration/batch-job-queue")
+      .andIsUnauthorized
+  }
+}

backend/app/src/test/kotlin/io/tolgee/batch/BatchJobTestUtil.kt

@@ -409,7 +409,7 @@ class BatchJobTestUtil(
 
   fun verifyProjectJobLockReleased() {
     waitFor(pollTime = 200, timeout = 1000) {
-      batchJobProjectLockingManager.getLockedForProject(testData.projectBuilder.self.id) == 0L
+      batchJobProjectLockingManager.getLockedForProject(testData.projectBuilder.self.id).isEmpty()
     }
   }