Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions .github/workflows/maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,27 @@ on:
- tagscanarymerge
- fixlabels
- interceptapis
- mlh-965
- mlh-498-use-spec-store-master
- tags_intg_test


jobs:
build:

runs-on: ubuntu-latest

# Add permissions for Docker
permissions:
contents: read
packages: write

steps:
- uses: actions/checkout@v3

# Set up Docker
- name: Set up Docker
uses: docker/setup-buildx-action@v2
with:
driver-opts: image=moby/buildkit:master
install: true

- name: Set up JDK 17
uses: actions/setup-java@v1
Expand All @@ -51,6 +61,12 @@ jobs:
- name: Print JDK version
run: java -version

# Verify Docker is available
- name: Verify Docker
run: |
docker --version
docker info

- name: Cache Maven packages
uses: actions/cache@v3
with:
Expand Down Expand Up @@ -78,6 +94,16 @@ jobs:
echo "build without dashboard"
chmod +x ./build.sh && ./build.sh

- name: Run Integration Tests
env:
# Configure Testcontainers for GitHub Actions
TESTCONTAINERS_RYUK_DISABLED: true
TESTCONTAINERS_CHECKS_DISABLE: true
DOCKER_HOST: unix:///var/run/docker.sock
run: |
echo "Running integration tests..."
chmod +x ./run-integration-tests.sh && ./run-integration-tests.sh

- name: Get Repository Name
run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
shell: bash
Expand Down
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ unzip -o keycloak-15.0.2.1.zip -d ~/.m2/repository/org

echo "Maven Building"

mvn -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist
mvn -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true install -Pdist

echo "[DEBUG listing distro/target"
ls distro/target
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public abstract class AbstractRedisService implements RedisService {
private static final String ATLAS_REDIS_LOCK_WAIT_TIME_MS = "atlas.redis.lock.wait_time.ms";
private static final String ATLAS_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = "atlas.redis.lock.watchdog_timeout.ms";
private static final String ATLAS_REDIS_LEASE_TIME_MS = "atlas.redis.lease_time.ms";
private static final String CHECK_SENTINELS_LIST = "atlas.redis.sentinel.check_list.enabled";
private static final int DEFAULT_REDIS_WAIT_TIME_MS = 15_000;
private static final int DEFAULT_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = 600_000;
private static final int DEFAULT_REDIS_LEASE_TIME_MS = 60_000;
Expand All @@ -48,6 +49,7 @@ public abstract class AbstractRedisService implements RedisService {
long waitTimeInMS;
long leaseTimeInMS;
long watchdogTimeoutInMS;
boolean checkSentinelsList;

// Inner class to track lock information
private static class LockInfo {
Expand Down Expand Up @@ -255,7 +257,7 @@ Config getCacheImplConfig() {
config.useSentinelServers()
.setClientName(ATLAS_METASTORE_SERVICE+"-redisCache")
.setReadMode(ReadMode.MASTER_SLAVE)
.setCheckSentinelsList(false)
.setCheckSentinelsList(checkSentinelsList)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Redis Sentinel Configuration Not Applied

The checkSentinelsList field is declared but not initialized from the atlas.redis.sentinel.check_list.enabled configuration. This means it always defaults to false when used in getCacheImplConfig(), preventing the intended configuration from taking effect.

Fix in Cursor Fix in Web

.setKeepAlive(true)
.setMasterConnectionMinimumIdleSize(10)
.setMasterConnectionPoolSize(20)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
package org.apache.atlas.service.redis;

import org.apache.atlas.AtlasException;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.redisson.Redisson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

@Component
@ConditionalOnAtlasProperty(property = "atlas.redis.service.impl")
@Profile("!local")
public class RedisServiceImpl extends AbstractRedisService{

private static final Logger LOG = LoggerFactory.getLogger(RedisServiceImpl.class);

@PostConstruct
public void init() throws AtlasException {
LOG.info("Initializing RedisServiceImpl");

redisClient = Redisson.create(getProdConfig());
redisCacheClient = Redisson.create(getCacheImplConfig());
LOG.debug("Sentinel redis client created successfully.");
Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
package org.apache.atlas.service.redis;

import org.apache.atlas.AtlasException;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.redisson.Redisson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;

@Component
@ConditionalOnAtlasProperty(property = "atlas.redis.service.impl")
@Profile("local")
public class RedisServiceLocalImpl extends AbstractRedisService {

private static final Logger LOG = LoggerFactory.getLogger(RedisServiceLocalImpl.class);

@PostConstruct
public void init() throws AtlasException {
LOG.info("Initializing RedisServiceLocalImpl");

redisClient = Redisson.create(getLocalConfig());
redisCacheClient = Redisson.create(getLocalConfig());
LOG.info("Local redis client created successfully.");
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1813,7 +1813,7 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-testng</artifactId>
<artifactId>surefire-junit-platform</artifactId>
<version>${surefire.version}</version>
</dependency>
</dependencies>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package org.apache.atlas.repository.integration;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
import org.apache.atlas.repository.store.graph.v2.EntityStream;
import org.apache.atlas.type.AtlasTypeRegistry;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class TestEntityStreamUtil {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();

public static class EntityCreationRequest {
private final String jsonRequest;
private List<AtlasEntity> entities;
private Map<String, AtlasEntityHeader> guidHeaderMap;

public EntityCreationRequest(String jsonRequest) {
this.jsonRequest = jsonRequest;
this.entities = new ArrayList<>();
this.guidHeaderMap = new HashMap<>();
}

public EntityStream toEntityStream(AtlasTypeRegistry typeRegistry) throws IOException {
JsonNode root = OBJECT_MAPPER.readTree(jsonRequest);
ArrayNode entitiesNode = (ArrayNode) root.get("entities");

for (JsonNode entityNode : entitiesNode) {
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo();
AtlasEntity entity = new AtlasEntity();

entity.setTypeName(entityNode.get("typeName").asText());

// Handle attributes
JsonNode attributesNode = entityNode.get("attributes");
Map<String, Object> attributes = new HashMap<>();
attributesNode.fields().forEachRemaining(field ->
attributes.put(field.getKey(), field.getValue().asText())
);
entity.setAttributes(attributes);

entityWithExtInfo.setEntity(entity);
entities.add(entity);
}

return new AtlasEntityStream(entities);
}

public void updateWithMutationResponse(EntityMutationResponse response) {
// Update entities with assigned GUIDs
for (AtlasEntityHeader header : response.getCreatedEntities()) {
guidHeaderMap.put(header.getGuid(), header);
}
}

public AtlasEntityHeaders toEntityHeaders() {
AtlasEntityHeaders headers = new AtlasEntityHeaders();
headers.setGuidHeaderMap(guidHeaderMap);
return headers;
}
}

public static EntityCreationRequest fromJson(String json) {
return new EntityCreationRequest(json);
}

}
124 changes: 124 additions & 0 deletions run-integration-tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/bin/bash
# run-integration-tests.sh

set -e

echo "============================================"
echo "Atlas Integration Tests Runner"
echo "============================================"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color
export JAVA_HOME=`/usr/libexec/java_home -v 17` # Set Java 17 as JAVA_HOME
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: Non-Portable Java Version Command

The script sets JAVA_HOME using /usr/libexec/java_home -v 17, a macOS-specific command. This causes failures on Linux and other non-macOS environments, making the integration tests non-portable.

Fix in Cursor Fix in Web

# Check if Docker is running
if ! docker info > /dev/null 2>&1; then
echo -e "${RED}Docker is not running. Please start Docker first.${NC}"
exit 1
fi

# Parse arguments
SKIP_BUILD=false
KEEP_CONTAINERS=false
DEBUG=false

while [[ "$#" -gt 0 ]]; do
case $1 in
--skip-build) SKIP_BUILD=true ;;
--keep-containers) KEEP_CONTAINERS=true ;;
--debug) DEBUG=true ;;
-h|--help)
echo "Usage: $0 [options]"
echo "Options:"
echo " --skip-build Skip building Atlas WAR and Docker image"
echo " --keep-containers Keep containers running after tests"
echo " --debug Enable debug logging"
echo " -h, --help Show this help message"
exit 0
;;
*) echo "Unknown parameter: $1"; exit 1 ;;
esac
shift
done

# Step 1: Build Docker image
echo -e "${YELLOW}Building Atlas Docker image...${NC}"
docker buildx build --load -t atlanhq/atlas:test .

if [ $? -ne 0 ]; then
echo -e "${RED}Failed to build Docker image${NC}"
exit 1
fi
echo -e "${GREEN}Docker image built successfully${NC}"

# Step 2: Clean up any existing test containers
echo -e "${YELLOW}Cleaning up existing test containers...${NC}"
docker rm -f atlas-test-zookeeper atlas-test-kafka atlas-test-cassandra \
atlas-test-elasticsearch atlas-test-redis atlas-test-atlas 2>/dev/null || true

# Step 3: Set test properties
export TESTCONTAINERS_REUSE_ENABLE=true
export TESTCONTAINERS_RYUK_DISABLED=$KEEP_CONTAINERS

if [ "$DEBUG" = true ]; then
export TESTCONTAINERS_DEBUG=true
MAVEN_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
fi

echo "Listing docker images"
docker image ls

# Step 4: Run integration tests
echo -e "${YELLOW}Running integration tests...${NC}"

if [ "$DEBUG" = true ]; then
mvn test -pl webapp -Dtest=AtlasDockerIntegrationTest \
-Dorg.slf4j.simpleLogger.defaultLogLevel=debug \
-Dorg.testcontainers.log.level=DEBUG -Dsurefire.useFile=false
else
mvn test -pl webapp -Dtest=AtlasDockerIntegrationTest -Dsurefire.useFile=false
fi

TEST_RESULT=$?

# Step 5: Collect logs if tests failed
if [ $TEST_RESULT -ne 0 ]; then
echo -e "${RED}Tests failed! Collecting logs...${NC}"

mkdir -p target/test-logs

# Get container logs
for container in $(docker ps -a --filter "name=atlas-test" --format "{{.Names}}"); do
echo "Collecting logs from $container..."
docker logs $container > "target/test-logs/${container}.log" 2>&1
done

echo -e "${YELLOW}Logs saved to target/test-logs/${NC}"
fi

# Step 6: Clean up containers if not keeping them
if [ "$KEEP_CONTAINERS" = false ]; then
echo -e "${YELLOW}Cleaning up test containers...${NC}"
docker rm -f $(docker ps -a --filter "name=atlas-test" --format "{{.Names}}") 2>/dev/null || true
else
echo -e "${YELLOW}Keeping containers running (--keep-containers flag set)${NC}"
echo "You can connect to Atlas at: http://localhost:21000"
echo "To stop containers manually, run:"
echo " docker rm -f \$(docker ps -a --filter 'name=atlas-test' --format '{{.Names}}')"
fi

# Step 7: Report results
if [ $TEST_RESULT -eq 0 ]; then
echo -e "${GREEN}============================================${NC}"
echo -e "${GREEN}Integration tests completed successfully!${NC}"
echo -e "${GREEN}============================================${NC}"
exit 0
else
echo -e "${RED}============================================${NC}"
echo -e "${RED}Integration tests failed!${NC}"
echo -e "${RED}Check target/test-logs for details${NC}"
echo -e "${RED}============================================${NC}"
exit 1
fi
Loading
Loading