CREATE TABLE fails when Iceberg is accessed from Adbc #595
Description
Hi!
I'm trying to use DuckDb (1.4.2) and the Iceberg extension in an Adbc application. I have hit a problem with a CREATE TABLE statement. Here is a simplified code that reproduces the problem:
int main()
{
AdbcError error = {};
AdbcDatabase database = {};
CHECK_ADBC(AdbcDatabaseNew(&database, &error));
CHECK_ADBC(AdbcDriverManagerDatabaseSetInitFunc(&database, &duckdb_adbc_init, &error));
CHECK_ADBC(AdbcDatabaseInit(&database, &error));
AdbcConnection connection = {};
CHECK_ADBC(AdbcConnectionNew(&connection, &error));
CHECK_ADBC(AdbcConnectionInit(&connection, &database, &error));
// Attach Iceberg catalog 'STAGING' and setup s3 secret
if (setup_step(&connection)) {
return EXIT_FAILURE;
}
{
AdbcStatement statement = {};
CHECK_ADBC(AdbcStatementNew(&connection, &statement, &error));
CHECK_ADBC(AdbcStatementSetSqlQuery(&statement, R"(DROP TABLE IF EXISTS STAGING.WOLF.LINEITEM)", &error));
CHECK_ADBC(AdbcStatementExecuteQuery(&statement, nullptr, nullptr, &error));
CHECK_ADBC(AdbcStatementRelease(&statement, &error));
}
{
AdbcStatement statement = {};
CHECK_ADBC(AdbcStatementNew(&connection, &statement, &error));
CHECK_ADBC(AdbcStatementSetSqlQuery(&statement, R"(CREATE TABLE STAGING.WOLF.LINEITEM AS SELECT * FROM read_parquet('... redacted ...'))", &error));
CHECK_ADBC(AdbcStatementExecuteQuery(&statement, nullptr, nullptr, &error)); // <-- this fails with "Invalid Configuration Error: Table LINEITEM already exists" error
CHECK_ADBC(AdbcStatementRelease(&statement, &error));
}
return 0;
}
The problem seems to be related to a fact that ICTableSet::CreateNewEntry is called twice.
First, it is called during the query planning (as part of AdbcStatementSetSqlQuery). Here is the relevant stack:
> duckdb.dll!duckdb::ICTableSet::CreateNewEntry(duckdb::ClientContext & context, duckdb::IRCatalog & catalog, duckdb::IRCSchemaEntry & schema, duckdb::CreateTableInfo & info) Line 114 C++
duckdb.dll!duckdb::IRCSchemaEntry::CreateTable(duckdb::IRCTransaction & irc_transaction, duckdb::ClientContext & context, duckdb::BoundCreateTableInfo & info) Line 40 C++
duckdb.dll!duckdb::IRCatalog::PlanCreateTableAs(duckdb::ClientContext & context, duckdb::PhysicalPlanGenerator & planner, duckdb::LogicalCreateTable & op, duckdb::PhysicalOperator & plan) Line 493 C++
duckdb.dll!duckdb::PhysicalPlanGenerator::CreatePlan(duckdb::LogicalCreateTable & op) Line 45 C++
duckdb.dll!duckdb::PhysicalPlanGenerator::CreatePlan(duckdb::LogicalOperator & op) Line 127 C++
duckdb.dll!duckdb::PhysicalPlanGenerator::PlanInternal(duckdb::LogicalOperator & op) Line 57 C++
duckdb.dll!duckdb::PhysicalPlanGenerator::ResolveAndPlan(duckdb::unique_ptr<duckdb::LogicalOperator,std::default_delete<duckdb::LogicalOperator>,1> op) Line 45 C++
duckdb.dll!duckdb::PhysicalPlanGenerator::Plan(duckdb::unique_ptr<duckdb::LogicalOperator,std::default_delete<duckdb::LogicalOperator>,1> op) Line 24 C++
duckdb.dll!duckdb::ClientContext::CreatePreparedStatementInternal(duckdb::ClientContextLock & lock, const std::string & query, duckdb::unique_ptr<duckdb::SQLStatement,std::default_delete<duckdb::SQLStatement>,1> statement, duckdb::optional_ptr<std::unordered_map<std::string,duckdb::BoundParameterData,duckdb::CaseInsensitiveStringHashFunction,duckdb::CaseInsensitiveStringEquality,std::allocator<std::pair<std::string const ,duckdb::BoundParameterData>>>,1> values) Line 409 C++
duckdb.dll!duckdb::ClientContext::CreatePreparedStatement(duckdb::ClientContextLock & lock, const std::string & query, duckdb::unique_ptr<duckdb::SQLStatement,std::default_delete<duckdb::SQLStatement>,1> statement, duckdb::optional_ptr<std::unordered_map<std::string,duckdb::BoundParameterData,duckdb::CaseInsensitiveStringHashFunction,duckdb::CaseInsensitiveStringEquality,std::allocator<std::pair<std::string const ,duckdb::BoundParameterData>>>,1> values, duckdb::PreparedStatementMode mode) Line 460 C++
duckdb.dll!duckdb::ClientContext::PrepareInternal::__l2::<lambda>() Line 710 C++
[External Code]
duckdb.dll!duckdb::ClientContext::RunFunctionInTransactionInternal(duckdb::ClientContextLock & lock, const std::function<void __cdecl(void)> & fun, bool requires_valid_transaction) Line 1161 C++
duckdb.dll!duckdb::ClientContext::PrepareInternal(duckdb::ClientContextLock & lock, duckdb::unique_ptr<duckdb::SQLStatement,std::default_delete<duckdb::SQLStatement>,1> statement) Line 709 C++
duckdb.dll!duckdb::ClientContext::Prepare(duckdb::unique_ptr<duckdb::SQLStatement,std::default_delete<duckdb::SQLStatement>,1> statement) Line 722 C++
duckdb.dll!duckdb::Connection::Prepare(duckdb::unique_ptr<duckdb::SQLStatement,std::default_delete<duckdb::SQLStatement>,1> statement) Line 160 C++
duckdb.dll!duckdb_prepare_extracted_statement(_duckdb_connection * connection, _duckdb_extracted_statements * extracted_statements, unsigned __int64 index, _duckdb_prepared_statement * * out_prepared_statement) Line 55 C++
duckdb.dll!duckdb_adbc::StatementSetSqlQuery(AdbcStatement * statement, const char * query, AdbcError * error) Line 1036 C++
duckdb.dll!AdbcStatementSetSqlQuery(AdbcStatement * statement, const char * query, AdbcError * error) Line 1414 C++
testduckdb.exe!main() Line 117 C++
[External Code]
The second time ICTableSet::CreateNewEntry is hit during AdbcStatementExecuteQuery. Again, here is the stack:
> duckdb.dll!duckdb::ICTableSet::CreateNewEntry(duckdb::ClientContext & context, duckdb::IRCatalog & catalog, duckdb::IRCSchemaEntry & schema, duckdb::CreateTableInfo & info) Line 114 C++
duckdb.dll!duckdb::IRCSchemaEntry::CreateTable(duckdb::IRCTransaction & irc_transaction, duckdb::ClientContext & context, duckdb::BoundCreateTableInfo & info) Line 40 C++
duckdb.dll!duckdb::IRCSchemaEntry::CreateTable(duckdb::CatalogTransaction transaction, duckdb::BoundCreateTableInfo & info) Line 66 C++
duckdb.dll!duckdb::Catalog::CreateTable(duckdb::CatalogTransaction transaction, duckdb::SchemaCatalogEntry & schema, duckdb::BoundCreateTableInfo & info) Line 142 C++
duckdb.dll!duckdb::PhysicalCreateTable::GetData(duckdb::ExecutionContext & context, duckdb::DataChunk & chunk, duckdb::OperatorSourceInput & input) Line 22 C++
duckdb.dll!duckdb::PipelineExecutor::GetData(duckdb::DataChunk & chunk, duckdb::OperatorSourceInput & input) Line 503 C++
duckdb.dll!duckdb::PipelineExecutor::FetchFromSource(duckdb::DataChunk & result) Line 528 C++
duckdb.dll!duckdb::PipelineExecutor::Execute(unsigned __int64 max_chunks) Line 228 C++
duckdb.dll!duckdb::PipelineTask::ExecuteTask(duckdb::TaskExecutionMode mode) Line 41 C++
duckdb.dll!duckdb::ExecutorTask::Execute(duckdb::TaskExecutionMode mode) Line 52 C++
duckdb.dll!duckdb::Executor::ExecuteTask(bool dry_run) Line 574 C++
duckdb.dll!duckdb::ClientContext::ExecuteTaskInternal(duckdb::ClientContextLock & lock, duckdb::BaseQueryResult & result, bool dry_run) Line 602 C++
duckdb.dll!duckdb::PendingQueryResult::ExecuteTaskInternal(duckdb::ClientContextLock & lock) Line 69 C++
duckdb.dll!duckdb::PendingQueryResult::ExecuteInternal(duckdb::ClientContextLock & lock) Line 75 C++
duckdb.dll!duckdb::PendingQueryResult::Execute() Line 95 C++
duckdb.dll!duckdb::PreparedStatement::Execute(std::unordered_map<std::string,duckdb::BoundParameterData,duckdb::CaseInsensitiveStringHashFunction,duckdb::CaseInsensitiveStringEquality,std::allocator<std::pair<std::string const ,duckdb::BoundParameterData>>> & named_values, bool allow_stream_result) Line 77 C++
duckdb.dll!duckdb_execute_prepared(_duckdb_prepared_statement * prepared_statement, duckdb_result * out_result) Line 423 C++
duckdb.dll!duckdb_adbc::StatementExecuteQuery(AdbcStatement * statement, ArrowArrayStream * out, __int64 * rows_affected, AdbcError * error) Line 944 C++
duckdb.dll!AdbcStatementExecuteQuery(AdbcStatement * statement, ArrowArrayStream * out, __int64 * rows_affected, AdbcError * error) Line 1286 C++
testduckdb.exe!main() Line 118 C++
[External Code]
I have not found any way around it, and it seems that CTAS is unusable in Adbc. Please advise if there is any fix/workaround.