Skip to content

Commit c220772

Browse files
shiv0408shiv0408
authored
Merge branch 'main' into dependabot/github_actions/actions/github-script-8
2 parents 320bc0e + a81866f commit c220772

File tree

50 files changed

+1790
-167
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1790
-167
lines changed

‎.github/workflows/maven-publish.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,14 @@ jobs:
2929
export-env: true
3030
env:
3131
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
32-
SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
33-
SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
32+
MAVEN_SNAPSHOTS_S3_REPO: op://opensearch-infra-secrets/maven-snapshots-s3/repo
33+
MAVEN_SNAPSHOTS_S3_ROLE: op://opensearch-infra-secrets/maven-snapshots-s3/role
34+
- name: Configure AWS credentials
35+
uses: aws-actions/configure-aws-credentials@v5
36+
with:
37+
role-to-assume: ${{ env.MAVEN_SNAPSHOTS_S3_ROLE }}
38+
aws-region: us-east-1
3439
- name: publish snapshots to maven
3540
run: |
3641
./gradlew publishPluginZipPublicationToSnapshotsRepository
37-
./gradlew publishShadowPublicationToSnapshotsRepository
42+
./gradlew publishShadowPublicationToSnapshotsRepository

‎build.gradle‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,7 @@ buildscript {
7474
mavenLocal()
7575
mavenCentral()
7676
maven { url "https://plugins.gradle.org/m2/" }
77-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
78-
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
77+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
7978
}
8079

8180
dependencies {
@@ -263,8 +262,7 @@ dependencies {
263262

264263
repositories {
265264
mavenLocal()
266-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
267-
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
265+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
268266
}
269267

270268
publishing {
@@ -292,10 +290,11 @@ publishing {
292290
repositories {
293291
maven {
294292
name = "Snapshots"
295-
url = "https://central.sonatype.com/repository/maven-snapshots/"
296-
credentials {
297-
username System.getenv("SONATYPE_USERNAME")
298-
password System.getenv("SONATYPE_PASSWORD")
293+
url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
294+
credentials(AwsCredentials) {
295+
accessKey = System.getenv("AWS_ACCESS_KEY_ID")
296+
secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
297+
sessionToken = System.getenv("AWS_SESSION_TOKEN")
299298
}
300299
}
301300
}

‎release-notes/opensearch-index-management.release-notes-3.3.0.0.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
## Version 3.3.0 Release Notes
2+
3+
Compatible with OpenSearch and OpenSearch Dashboards version 3.3.0
4+
5+
### Features
6+
* Add support in SM Plugin to delete snapshots created manually ([#1452](https://github.com/opensearch-project/index-management/pull/1452))
7+
8+
### Enhancements
9+
* Using Scripted Avg Class in AvgAggregationBuilder in place of double ([#1460](https://github.com/opensearch-project/index-management/pull/1460))
10+
* Do not include global state in snapshot step ([#1480](https://github.com/opensearch-project/index-management/pull/1480))
11+
12+
### Bug Fixes
13+
* Fix the build ([#1491](https://github.com/opensearch-project/index-management/pull/1491))
14+
* Fix: Update System.env syntax for Gradle 9 compatibility ([#1474](https://github.com/opensearch-project/index-management/pull/1474))
15+
16+
### Infrastructure
17+
* Dependabot: bump 1password/load-secrets-action from 2 to 3 ([#1473](https://github.com/opensearch-project/index-management/pull/1473))
18+
19+
### Maintenance
20+
* [AUTO] Increment version to 3.3.0-SNAPSHOT ([#1467](https://github.com/opensearch-project/index-management/pull/1467))

‎spi/build.gradle‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,15 @@ repositories {
4949
mavenLocal()
5050
mavenCentral()
5151
maven { url "https://plugins.gradle.org/m2/" }
52-
maven { url "https://central.sonatype.com/repository/maven-snapshots/" }
53-
maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
52+
maven { url "https://ci.opensearch.org/ci/dbc/snapshots/maven/" }
5453
}
5554

5655
configurations.configureEach {
5756
if (it.state != Configuration.State.UNRESOLVED) return
5857
resolutionStrategy {
5958
force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
6059
force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
60+
force "com.google.errorprone:error_prone_annotations:${versions.error_prone_annotations}"
6161
}
6262
}
6363

@@ -71,7 +71,6 @@ dependencies {
7171
compileOnly "org.opensearch:common-utils:${common_utils_version}"
7272

7373
testImplementation "org.opensearch.test:framework:${opensearch_version}"
74-
testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
7574
}
7675

7776
idea.module {
@@ -118,10 +117,11 @@ publishing {
118117
}
119118
maven {
120119
name = "Snapshots"
121-
url = "https://central.sonatype.com/repository/maven-snapshots/"
122-
credentials {
123-
username System.getenv("SONATYPE_USERNAME")
124-
password System.getenv("SONATYPE_PASSWORD")
120+
url = System.getenv("MAVEN_SNAPSHOTS_S3_REPO")
121+
credentials(AwsCredentials) {
122+
accessKey = System.getenv("AWS_ACCESS_KEY_ID")
123+
secretKey = System.getenv("AWS_SECRET_ACCESS_KEY")
124+
sessionToken = System.getenv("AWS_SESSION_TOKEN")
125125
}
126126
}
127127
}

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ManagedIndexCoordinator.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -337,10 +337,10 @@ class ManagedIndexCoordinator(
337337
if (clusterState.metadata.hasIndex(indexName)) {
338338
val indexMetadata = clusterState.metadata.index(indexName)
339339
val autoManage =
340-
if (AUTO_MANAGE.get(indexMetadata.settings)) {
341-
true
342-
} else {
340+
if (indexMetadata.settings.get(AUTO_MANAGE.key).isNullOrBlank()) {
343341
LegacyOpenDistroManagedIndexSettings.AUTO_MANAGE.get(indexMetadata.settings)
342+
} else {
343+
AUTO_MANAGE.get(indexMetadata.settings)
344344
}
345345
if (autoManage) {
346346
val isHiddenIndex =

‎src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SMRunner.kt‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -103,17 +103,20 @@ object SMRunner :
103103

104104
// creation, deletion workflow have to be executed sequentially,
105105
// because they are sharing the same metadata document.
106-
SMStateMachine(client, job, metadata, settings, threadPool, indicesManager)
106+
val stateMachine = SMStateMachine(client, job, metadata, settings, threadPool, indicesManager)
107107
.handlePolicyChange()
108-
.currentState(metadata.creation.currentState)
109-
.next(creationTransitions)
110-
.apply {
111-
val deleteMetadata = metadata.deletion
112-
if (deleteMetadata != null) {
113-
this.currentState(deleteMetadata.currentState)
114-
.next(deletionTransitions)
115-
}
116-
}
108+
109+
// Execute creation workflow if it exists
110+
if (metadata.creation != null) {
111+
stateMachine.currentState(metadata.creation.currentState)
112+
.next(creationTransitions)
113+
}
114+
115+
// Execute deletion workflow if it exists
116+
if (metadata.deletion != null) {
117+
stateMachine.currentState(metadata.deletion.currentState)
118+
.next(deletionTransitions)
119+
}
117120
} finally {
118121
if (!releaseLockForScheduledJob(context, lock)) {
119122
log.error("Could not release lock [${lock.lockId}] for ${job.id}.")
@@ -155,13 +158,14 @@ object SMRunner :
155158
id = smPolicyNameToMetadataDocId(smDocIdToPolicyName(job.id)),
156159
policySeqNo = job.seqNo,
157160
policyPrimaryTerm = job.primaryTerm,
158-
creation =
159-
SMMetadata.WorkflowMetadata(
160-
SMState.CREATION_START,
161-
SMMetadata.Trigger(
162-
time = job.creation.schedule.getNextExecutionTime(now),
163-
),
164-
),
161+
creation = job.creation?.let {
162+
SMMetadata.WorkflowMetadata(
163+
SMState.CREATION_START,
164+
SMMetadata.Trigger(
165+
time = job.creation.schedule.getNextExecutionTime(now),
166+
),
167+
)
168+
},
165169
deletion =
166170
job.deletion?.let {
167171
SMMetadata.WorkflowMetadata(

‎src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SMUtils.kt‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import org.opensearch.action.get.GetRequest
1717
import org.opensearch.action.get.GetResponse
1818
import org.opensearch.action.index.IndexRequest
1919
import org.opensearch.action.index.IndexResponse
20+
import org.opensearch.common.regex.Regex
2021
import org.opensearch.common.time.DateFormatter
2122
import org.opensearch.common.time.DateFormatters
2223
import org.opensearch.common.unit.TimeValue
@@ -216,9 +217,11 @@ fun List<SnapshotInfo>.filterBySMPolicyInSnapshotMetadata(policyName: String): L
216217
*/
217218
suspend fun Client.getSnapshots(name: String, repo: String): List<SnapshotInfo> {
218219
try {
220+
val patterns = name.split(",").map { it.trim() }.toTypedArray()
221+
219222
val req =
220223
GetSnapshotsRequest()
221-
.snapshots(arrayOf(name))
224+
.snapshots(patterns)
222225
.repository(repo)
223226
val res: GetSnapshotsResponse = admin().cluster().suspendUntil { getSnapshots(req, it) }
224227
return res.snapshots
@@ -240,7 +243,7 @@ suspend fun Client.getSnapshots(
240243
snapshotMissingMsg: String?,
241244
exceptionMsg: String,
242245
): GetSnapshotsResult {
243-
val snapshots =
246+
var snapshots =
244247
try {
245248
getSnapshots(
246249
name,
@@ -257,7 +260,19 @@ suspend fun Client.getSnapshots(
257260
cause = ex,
258261
)
259262
return GetSnapshotsResult(true, emptyList(), metadataBuilder)
260-
}.filterBySMPolicyInSnapshotMetadata(job.policyName)
263+
}
264+
265+
// Parse CSV patterns from name and implement pattern-based filtering
266+
val patterns = name.split(",").map { it.trim() }
267+
val policyNamePattern = "${job.policyName}*"
268+
// Filter other snapshots by policy metadata
269+
val otherPatternSnapshots = snapshots.filter { snapshot ->
270+
patterns.any { pattern ->
271+
pattern != policyNamePattern && Regex.simpleMatch(pattern, snapshot.snapshotId().name)
272+
}
273+
}
274+
val policySnapshots = snapshots.filterBySMPolicyInSnapshotMetadata(job.policyName)
275+
snapshots = policySnapshots + otherPatternSnapshots
261276

262277
return GetSnapshotsResult(false, snapshots, metadataBuilder)
263278
}

‎src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/engine/SMStateMachine.kt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ class SMStateMachine(
162162
val retry =
163163
when (result.workflowType) {
164164
WorkflowType.CREATION -> {
165-
metadata.creation.retry
165+
metadata.creation?.retry
166166
}
167167
WorkflowType.DELETION -> {
168168
metadata.deletion?.retry
@@ -249,7 +249,11 @@ class SMStateMachine(
249249
val metadataToSave =
250250
SMMetadata.Builder(metadata)
251251
.setSeqNoPrimaryTerm(job.seqNo, job.primaryTerm)
252-
.setNextCreationTime(job.creation.schedule.getNextExecutionTime(now))
252+
253+
val creation = job.creation
254+
creation?.let {
255+
metadataToSave.setNextCreationTime(creation.schedule.getNextExecutionTime(now))
256+
}
253257

254258
val deletion = job.deletion
255259
deletion?.let {

‎src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/engine/states/creation/CreatingState.kt‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,14 @@ object CreatingState : State {
3838
SMMetadata.Builder(metadata)
3939
.workflow(WorkflowType.CREATION)
4040

41-
var snapshotName: String? = metadata.creation.started?.first()
41+
if (job.creation == null) {
42+
log.warn("Policy creation config becomes null before trying to create snapshot. Reset.")
43+
return SMResult.Fail(
44+
metadataBuilder.resetCreation(), WorkflowType.CREATION, true,
45+
)
46+
}
47+
48+
var snapshotName: String? = metadata.creation?.started?.first()
4249

4350
// Check if there's already a snapshot created by SM in current execution period.
4451
// So that this State can be executed idempotent.
@@ -54,8 +61,8 @@ object CreatingState : State {
5461
}
5562
val getSnapshots = getSnapshotsResult.snapshots
5663

57-
val latestExecutionStartTime = job.creation.schedule.getPeriodStartingAt(null).v1()
58-
snapshotName = checkCreatedSnapshots(latestExecutionStartTime, getSnapshots)
64+
val latestExecutionStartTime = job.creation?.schedule?.getPeriodStartingAt(null)?.v1()
65+
snapshotName = latestExecutionStartTime?.let { checkCreatedSnapshots(it, getSnapshots) }
5966
if (snapshotName != null) {
6067
log.info("Already created snapshot [$snapshotName] during this execution period starting at $latestExecutionStartTime.")
6168
metadataBuilder.setLatestExecution(

‎src/main/kotlin/org/opensearch/indexmanagement/snapshotmanagement/engine/states/creation/CreationConditionMetState.kt‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@ import org.opensearch.indexmanagement.snapshotmanagement.engine.states.State
1111
import org.opensearch.indexmanagement.snapshotmanagement.engine.states.WorkflowType
1212
import org.opensearch.indexmanagement.snapshotmanagement.model.SMMetadata
1313
import org.opensearch.indexmanagement.snapshotmanagement.tryUpdatingNextExecutionTime
14+
import java.time.Instant.now
1415

1516
object CreationConditionMetState : State {
1617
override val continuous = true
1718

19+
@Suppress("ReturnCount")
1820
override suspend fun execute(context: SMStateMachine): SMResult {
1921
val job = context.job
2022
val metadata = context.metadata
@@ -24,15 +26,35 @@ object CreationConditionMetState : State {
2426
SMMetadata.Builder(metadata)
2527
.workflow(WorkflowType.CREATION)
2628

27-
val nextCreationTime = metadata.creation.trigger.time
28-
val updateNextTimeResult =
29-
tryUpdatingNextExecutionTime(
30-
metadataBuilder, nextCreationTime, job.creation.schedule, WorkflowType.CREATION, log,
29+
if (job.creation == null) {
30+
log.warn("Policy creation config becomes null before trying to create snapshot. Reset.")
31+
return SMResult.Fail(
32+
metadataBuilder.resetCreation(), WorkflowType.CREATION, true,
3133
)
32-
if (!updateNextTimeResult.updated) {
33-
return SMResult.Stay(metadataBuilder)
3434
}
35-
metadataBuilder = updateNextTimeResult.metadataBuilder
35+
36+
// if job.creation != null, then metadata.creation.trigger.time should already be
37+
// initialized or handled in handlePolicyChange before executing this state.
38+
val nextCreationTime = if (metadata.creation == null) {
39+
val nextTime = job.creation.schedule.getNextExecutionTime(now())
40+
nextTime?.let { metadataBuilder.setNextCreationTime(it) }
41+
nextTime
42+
} else {
43+
metadata.creation.trigger.time
44+
}
45+
46+
nextCreationTime?.let { creationTime ->
47+
job.creation.schedule.let { schedule ->
48+
val updateNextTimeResult =
49+
tryUpdatingNextExecutionTime(
50+
metadataBuilder, creationTime, schedule, WorkflowType.CREATION, log,
51+
)
52+
if (!updateNextTimeResult.updated) {
53+
return SMResult.Stay(metadataBuilder)
54+
}
55+
metadataBuilder = updateNextTimeResult.metadataBuilder
56+
}
57+
}
3658

3759
return SMResult.Next(metadataBuilder)
3860
}

0 commit comments

Comments
 (0)