Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 98 additions & 81 deletions api/src/org/labkey/api/util/DebugInfoDumper.java
Original file line number Diff line number Diff line change
Expand Up @@ -316,119 +316,136 @@ static boolean justWaiting(StackTraceElement[] stack)
// OK probably just waiting for work. We could check for common tomcat/labkey patterns here to be more conservative.
}

/** Prevent reentrancy when we get SQLException trying to grab locks from Postgres */
private static final ThreadLocal<Boolean> DUMPING_THREADS = ThreadLocal.withInitial(() -> false);

/**
* Writes the thread dump into threads.txt
* */
public static synchronized void dumpThreads(LoggerWriter logWriter)
{
logWriter.debug("*********************************************");
logWriter.debug("Starting thread dump - " + LocalDateTime.now());
long used = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
logWriter.debug("Heap usage at " + DecimalFormat.getPercentInstance().format(((double)used / (double)max)) + " - " +
FileUtils.byteCountToDisplaySize(used) + " from a max of " +
FileUtils.byteCountToDisplaySize(max) + " (" + DecimalFormat.getInstance().format(used) + " / " + DecimalFormat.getInstance().format(max) + " bytes)");

OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
if (osBean != null)
if (DUMPING_THREADS.get() == true)
{
DecimalFormat f3 = new DecimalFormat("0.000");

if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean)
{
logWriter.debug("Total OS memory (bytes): " + DecimalFormat.getInstance().format(sunOsBean.getTotalMemorySize()));
logWriter.debug("Free OS memory (bytes): " + DecimalFormat.getInstance().format(sunOsBean.getFreeMemorySize()));
logWriter.debug("OS CPU load: " + f3.format(sunOsBean.getCpuLoad()));
logWriter.debug("JVM CPU load: " + f3.format(sunOsBean.getProcessCpuLoad()));
}
logWriter.debug("CPU count: " + osBean.getAvailableProcessors());
return;
}

logWriter.debug("*********************************************");
try
{
DUMPING_THREADS.set(true);

Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
var spidsByThread = ConnectionWrapper.getSPIDsForThreads();
logWriter.debug("*********************************************");
logWriter.debug("Starting thread dump - " + LocalDateTime.now());
long used = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed();
long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
logWriter.debug("Heap usage at " + DecimalFormat.getPercentInstance().format(((double) used / (double) max)) + " - " +
FileUtils.byteCountToDisplaySize(used) + " from a max of " +
FileUtils.byteCountToDisplaySize(max) + " (" + DecimalFormat.getInstance().format(used) + " / " + DecimalFormat.getInstance().format(max) + " bytes)");

OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
if (osBean != null)
{
DecimalFormat f3 = new DecimalFormat("0.000");

ArrayList<Thread> threadsToDump = new ArrayList<>();
ArrayList<Thread> boringThreads = new ArrayList<>();
if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean)
{
logWriter.debug("Total OS memory (bytes): " + DecimalFormat.getInstance().format(sunOsBean.getTotalMemorySize()));
logWriter.debug("Free OS memory (bytes): " + DecimalFormat.getInstance().format(sunOsBean.getFreeMemorySize()));
logWriter.debug("OS CPU load: " + f3.format(sunOsBean.getCpuLoad()));
logWriter.debug("JVM CPU load: " + f3.format(sunOsBean.getProcessCpuLoad()));
}
logWriter.debug("CPU count: " + osBean.getAvailableProcessors());
}

for (Thread thread : stackTraces.keySet())
{
Set<Integer> spids = Objects.requireNonNullElse(spidsByThread.get(thread), Set.of());
var stack = stackTraces.get(thread);
logWriter.debug("*********************************************");

if (spids.isEmpty() && justWaiting(stack))
boringThreads.add(thread);
else
threadsToDump.add(thread);
}
Map<Thread, StackTraceElement[]> stackTraces = Thread.getAllStackTraces();
var spidsByThread = ConnectionWrapper.getSPIDsForThreads();

if (!threadsToDump.isEmpty())
{
logWriter.debug("");
logWriter.debug(" ----- active threads -----");
ArrayList<Thread> threadsToDump = new ArrayList<>();
ArrayList<Thread> boringThreads = new ArrayList<>();

threadsToDump.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER));
for (Thread thread : threadsToDump)
for (Thread thread : stackTraces.keySet())
{
dumpOneThread(thread, logWriter, stackTraces, spidsByThread);
}
}
Set<Integer> spids = Objects.requireNonNullElse(spidsByThread.get(thread), Set.of());
var stack = stackTraces.get(thread);

if (!boringThreads.isEmpty())
{
logWriter.debug("");
logWriter.debug(" ----- waiting threads -----");
if (spids.isEmpty() && justWaiting(stack))
boringThreads.add(thread);
else
threadsToDump.add(thread);
}

boringThreads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER));
for (Thread thread : boringThreads)
if (!threadsToDump.isEmpty())
{
dumpOneThread(thread, logWriter, stackTraces, spidsByThread);
logWriter.debug("");
logWriter.debug(" ----- active threads -----");

threadsToDump.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER));
for (Thread thread : threadsToDump)
{
dumpOneThread(thread, logWriter, stackTraces, spidsByThread);
}
}
}

logWriter.debug("*********************************************");
logWriter.debug("Completed thread dump");
logWriter.debug("*********************************************");
if (!boringThreads.isEmpty())
{
logWriter.debug("");
logWriter.debug(" ----- waiting threads -----");

for (DbScope dbScope : DbScope.getDbScopes())
{
dbScope.logCurrentConnectionState(logWriter);
}
boringThreads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER));
for (Thread thread : boringThreads)
{
dumpOneThread(thread, logWriter, stackTraces, spidsByThread);
}
}

if (ConnectionWrapper.getActiveConnectionCount() > 0)
{
logWriter.debug("*********************************************");
logWriter.debug("Start dump of all open connections");
logWriter.debug("Completed thread dump");
logWriter.debug("*********************************************");
ConnectionWrapper.dumpOpenConnections(logWriter, null);
logWriter.debug("*********************************************");
logWriter.debug("Completed dump of all open connections");
logWriter.debug("*********************************************");
}

// GitHub Issue 713: Automatically include PG locks and active queries in thread dumps
UserSchema schema = QueryService.get().getUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot(), BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME);
// Schema won't exist on SQLServer
if (schema != null)
{
try
{
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks");
}
catch (RuntimeException e)
for (DbScope dbScope : DbScope.getDbScopes())
{
logWriter.debug("Failed to write Postgres locks table:" + e);
dbScope.logCurrentConnectionState(logWriter);
}
try

if (ConnectionWrapper.getActiveConnectionCount() > 0)
{
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity");
logWriter.debug("*********************************************");
logWriter.debug("Start dump of all open connections");
logWriter.debug("*********************************************");
ConnectionWrapper.dumpOpenConnections(logWriter, null);
logWriter.debug("*********************************************");
logWriter.debug("Completed dump of all open connections");
logWriter.debug("*********************************************");
}
catch (RuntimeException e)

// GitHub Issue 713: Automatically include PG locks and active queries in thread dumps
UserSchema schema = QueryService.get().getUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot(), BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME);
// Schema won't exist on SQLServer
if (schema != null)
{
logWriter.debug("Failed to write Postgres activity table:" + e);
try
{
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_LOCKS_TABLE_NAME, "Postgres locks");
}
catch (RuntimeException e)
{
logWriter.debug("Failed to write Postgres locks table:" + e);
}
try
{
writeTable(logWriter, schema, BasePostgreSqlDialect.POSTGRES_STAT_ACTIVITY_TABLE_NAME, "Postgres activity");
}
catch (RuntimeException e)
{
logWriter.debug("Failed to write Postgres activity table:" + e);
}
}
}
finally
{
DUMPING_THREADS.set(false);
}
}

private static void writeTable(LoggerWriter logWriter, UserSchema schema, String tableName, String header)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ public interface IVisualizationSourceQuery

boolean isSkipVisitJoin();

boolean isVisitTagQuery();

/**
* True if any select or aggregate requires a left join explicitly. This is an override for any columns
* that might require some form of an INNER JOIN.
Expand Down
22 changes: 0 additions & 22 deletions study/src/org/labkey/study/query/StudyQuerySchema.java
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,6 @@ public class StudyQuerySchema extends UserSchema implements UserSchema.HasContex
public static final String VISIT_TAG_TABLE_NAME = "VisitTag";
public static final String VISIT_TAG_MAP_TABLE_NAME = "VisitTagMap";
public static final String VISIT_ALIASES = "VisitAliases";
public static final String VISUALIZATION_VISIT_TAG_TABLE_NAME = "VisualizationVisitTag";
public static final String VISIT_MAP_TABLE_NAME = "VisitMap";

public static final String STUDY_DATA_TABLE_NAME = "StudyData";
Expand Down Expand Up @@ -642,27 +641,6 @@ public TableInfo createTable(String name, ContainerFilter cf)
{
return new VisitMapTable(this, cf);
}
if (name.startsWith(VISUALIZATION_VISIT_TAG_TABLE_NAME))
{
// Name is encoded with useProtocolDay boolean, tagName, and altQueryName
String params = name.substring(VISUALIZATION_VISIT_TAG_TABLE_NAME.length());
boolean useProtocolDay;
if (params.startsWith("-true"))
{
params = params.substring(params.indexOf("-true") + 6);
useProtocolDay = true;
}
else
{
params = params.substring(params.indexOf("-false") + 7);
useProtocolDay = false;
}
int hyphenIndex = params.indexOf("-");
String tagName = hyphenIndex > -1 ? params.substring(0, hyphenIndex) : params;
String altQueryName = hyphenIndex > -1 ? params.substring(hyphenIndex + 1) : null;

return new VisualizationVisitTagTable(this, cf, getStudy(), getUser(), tagName, useProtocolDay, altQueryName);
}

// Might be a dataset
DatasetDefinition dsd = getDatasetDefinitionByQueryName(name);
Expand Down
104 changes: 0 additions & 104 deletions study/src/org/labkey/study/query/VisualizationVisitTagTable.java

This file was deleted.

Loading