dataforge/fn_registry
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(playground): DuckDB adapter para TQL->SQL execute (issue 0080)

Browse Source 
Cierra 0080 fase 11. tql_duckdb.{h,cpp} es adapter opcional gated por
build flag FN_TQL_DUCKDB=ON. Default OFF — playground sin deps DuckDB.

Funcionalidad:
- tql_duckdb::execute(sql, params, tables) -> Result con StageOutput
  materializado. Abre DuckDB :memory:, registra TableInputs via
  CREATE TABLE + INSERT batched (1000 rows/batch), prepare + bind
  params via duckdb_bind_varchar, execute_prepared, materializa
  resultado via duckdb_value_varchar + duckdb_free.
- type_from_duckdb mapeo DuckDB type -> ColumnType.
- CMakeLists.txt: option(FN_TQL_DUCKDB) + condicional add a sources
  + link duckdb_vendored + copy runtime.
- data_table.cpp Ask AI modal: ifdef FN_TQL_DUCKDB para status message
  apropiado en SQL apply.
- self_test.cpp: 4 round-trip tests gated por FN_TQL_DUCKDB:
  stage0 SELECT, group+count, filter Op::Eq, sum aggregation
  (verifica sum_n(go)=30).

Tests:
- 603 passed sin FN_TQL_DUCKDB (default OFF).
- 618 passed con FN_TQL_DUCKDB=ON (round-trip TQL emit -> DuckDB
  execute -> match compute_stage pure).
- e2e linux + windows cross-build OK ambos modos.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Egutierrez
2026-05-13 00:58:18 +02:00
parent e3c8979e8d
commit 2ca973fb7c
5 changed files with 384 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cpp/apps/primitives_gallery/playground/tables/CMakeLists.txt
+25 -5
View File
@@ -1,5 +1,8 @@
# Tables playground (cpp_apps.md / playgrounds.md). NO se indexa.
add_imgui_app(tables_playground
# Build flag FN_TQL_DUCKDB=ON activa el adapter tql_duckdb (issue 0080).
option(FN_TQL_DUCKDB "Enable DuckDB SQL execution adapter for tables playground" OFF)
set(_TABLES_SRC
main.cpp
data_table.cpp
data_table_logic.cpp
@@ -9,10 +12,7 @@ add_imgui_app(tables_playground
tql_to_sql.cpp
viz.cpp
)
target_link_libraries(tables_playground PRIVATE lua54 implot)
# Self-test E2E (logica pura + lua_engine + tql).
add_executable(tables_playground_self_test
set(_TABLES_TEST_SRC
self_test.cpp
data_table_logic.cpp
llm_anthropic.cpp
@@ -20,8 +20,28 @@ add_executable(tables_playground_self_test
tql.cpp
tql_to_sql.cpp
)
if(FN_TQL_DUCKDB)
list(APPEND _TABLES_SRC tql_duckdb.cpp)
list(APPEND _TABLES_TEST_SRC tql_duckdb.cpp)
endif()
add_imgui_app(tables_playground ${_TABLES_SRC})
target_link_libraries(tables_playground PRIVATE lua54 implot)
if(FN_TQL_DUCKDB)
target_compile_definitions(tables_playground PRIVATE FN_TQL_DUCKDB=1)
target_link_libraries(tables_playground PRIVATE duckdb_vendored)
duckdb_copy_runtime(tables_playground)
endif()
# Self-test E2E (logica pura + lua_engine + tql).
add_executable(tables_playground_self_test ${_TABLES_TEST_SRC})
target_include_directories(tables_playground_self_test PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_SOURCE_DIR}/functions
)
target_link_libraries(tables_playground_self_test PRIVATE lua54)
if(FN_TQL_DUCKDB)
target_compile_definitions(tables_playground_self_test PRIVATE FN_TQL_DUCKDB=1)
target_link_libraries(tables_playground_self_test PRIVATE duckdb_vendored)
duckdb_copy_runtime(tables_playground_self_test)
endif()
cpp/apps/primitives_gallery/playground/tables/data_table.cpp
+13 -1
View File
@@ -5,6 +5,9 @@
#include "lua_engine.h"
#include "tql.h"
#include "tql_to_sql.h"
#ifdef FN_TQL_DUCKDB
# include "tql_duckdb.h"
#endif
#include "viz.h"
#include <algorithm>
@@ -3341,8 +3344,17 @@ void render(const char* id,
U.ask_status = "Apply failed.";
}
} else {
// SQL apply: requires DuckDB adapter (no v1).
#ifdef FN_TQL_DUCKDB
// SQL apply: ejecutar via tql_duckdb sobre TableInputs activas.
// Para tablas en memoria construimos un TableInput basico desde
// active_headers/types. v1 no recupera cells originales aqui;
// reportamos solo error si fallo. Caller real deberia pasar
// tables() del render scope. Sin esto, marcamos status info.
U.ask_status = "SQL execute disponible (FN_TQL_DUCKDB ON). "
"Integracion full pendiente: usar tql_duckdb::execute desde caller.";
#else
U.ask_status = "SQL execute requires FN_TQL_DUCKDB build flag.";
#endif
}
}
ImGui::EndDisabled();
cpp/apps/primitives_gallery/playground/tables/self_test.cpp
+86
View File
@@ -11,6 +11,9 @@
#include "lua_engine.h"
#include "tql.h"
#include "tql_to_sql.h"
#ifdef FN_TQL_DUCKDB
# include "tql_duckdb.h"
#endif
#include <cmath>
#include <cstdio>
@@ -2830,6 +2833,89 @@ return {
putenv((char*)"FN_LLM_MOCK_RESPONSE=");
}
#ifdef FN_TQL_DUCKDB
// === phase11 round-trip TQL emit -> DuckDB execute -> match compute_stage ===
{
// Setup tabla "users" con 5 rows.
const char* cells[] = {
"go", "10",
"go", "20",
"py", "30",
"py", "40",
"cpp", "50",
};
TableInput t;
t.name = "users";
t.headers = {"lang", "n"};
t.types = {ColumnType::String, ColumnType::Int};
t.cells = cells;
t.rows = 5;
t.cols = 2;
std::vector<TableInput> tables = {t};
// Caso A: stage 0 simple SELECT
{
State st;
st.stages.resize(1);
auto e = tql_to_sql::emit_sql(st, tables);
check(e.error.empty(), "phase11 rt: stage0 emit OK");
auto r = tql_duckdb::execute(e.sql, e.params, tables);
check(r.error.empty(), "phase11 rt: stage0 execute OK");
check(r.out.rows == 5, "phase11 rt: stage0 5 rows");
check(r.out.cols == 2, "phase11 rt: stage0 2 cols");
}
// Caso B: stage 1 group by lang + count
{
State st;
st.stages.resize(2);
st.stages[1].breakouts.push_back("lang");
st.stages[1].aggregations.push_back({AggFn::Count});
st.active_stage = 1;
auto e = tql_to_sql::emit_sql(st, tables);
check(e.error.empty(), "phase11 rt: group emit OK");
auto r = tql_duckdb::execute(e.sql, e.params, tables);
check(r.error.empty(), "phase11 rt: group execute OK");
check(r.out.rows == 3, "phase11 rt: 3 grupos (go/py/cpp)");
check(r.out.cols == 2, "phase11 rt: cols = lang + count");
}
// Caso C: filter Op::Eq
{
State st;
st.stages.resize(1);
st.stages[0].filters.push_back({0, Op::Eq, "go"});
auto e = tql_to_sql::emit_sql(st, tables);
check(e.error.empty(), "phase11 rt: filter emit OK");
auto r = tql_duckdb::execute(e.sql, e.params, tables);
check(r.error.empty(), "phase11 rt: filter execute OK");
check(r.out.rows == 2, "phase11 rt: filter -> 2 rows go");
}
// Caso D: aggregation sum
{
State st;
st.stages.resize(2);
st.stages[1].breakouts.push_back("lang");
st.stages[1].aggregations.push_back({AggFn::Sum, "n"});
st.active_stage = 1;
auto e = tql_to_sql::emit_sql(st, tables);
check(e.error.empty(), "phase11 rt: sum emit OK");
auto r = tql_duckdb::execute(e.sql, e.params, tables);
check(r.error.empty(), "phase11 rt: sum execute OK");
check(r.out.rows == 3, "phase11 rt: sum 3 grupos");
// Verifica que sum_n para "go" es 30 (10+20)
bool found_go_30 = false;
for (int rr = 0; rr < r.out.rows; ++rr) {
std::string lang = r.out.cells[rr * 2 + 0];
std::string sum = r.out.cells[rr * 2 + 1];
if (lang == "go" && (sum == "30" || sum == "30.0")) found_go_30 = true;
}
check(found_go_30, "phase11 rt: sum_n(go) = 30");
}
}
#endif
std::printf("\n=== %d passed, %d failed ===\n", passed, failed);
return failed == 0 ? 0 : 1;
}
cpp/apps/primitives_gallery/playground/tables/tql_duckdb.cpp
+231
View File
@@ -0,0 +1,231 @@
// tql_duckdb.cpp — DuckDB adapter para ejecutar SQL emitido por tql_to_sql.
// Compilado solo si FN_TQL_DUCKDB define. Ver issue 0080.
#include "tql_duckdb.h"
#ifdef FN_TQL_DUCKDB
#include "duckdb.h"
#include <chrono>
#include <cstdio>
#include <cstring>
#include <string>
#include <vector>
namespace tql_duckdb {
using namespace data_table;
namespace {
// SQL identifier quote (mismo patron que tql_to_sql).
std::string sql_ident(const std::string& name) {
std::string out;
out.reserve(name.size() + 4);
out += '"';
for (char c : name) {
if (c == '"') out += "\"\"";
else out += c;
}
out += '"';
return out;
}
const char* duckdb_type_for(ColumnType t) {
switch (t) {
case ColumnType::Int: return "BIGINT";
case ColumnType::Float: return "DOUBLE";
case ColumnType::Bool: return "BOOLEAN";
case ColumnType::Date: return "VARCHAR"; // se almacena ISO como texto v1
case ColumnType::Json: return "VARCHAR";
default: return "VARCHAR";
}
}
// SQL literal escape para string.
std::string lit_str(const char* s) {
std::string out = "'";
for (const char* p = s ? s : ""; *p; ++p) {
if (*p == '\'') out += "''";
else out += *p;
}
out += "'";
return out;
}
bool create_and_load(duckdb_connection cn, const TableInput& t, std::string& err) {
// CREATE TABLE
std::string ddl = "CREATE TABLE " + sql_ident(t.name) + " (";
for (size_t i = 0; i < t.headers.size(); ++i) {
if (i > 0) ddl += ", ";
ColumnType ct = (i < t.types.size()) ? t.types[i] : ColumnType::String;
ddl += sql_ident(t.headers[i]) + " " + duckdb_type_for(ct);
}
ddl += ");";
duckdb_result rr;
if (duckdb_query(cn, ddl.c_str(), &rr) == DuckDBError) {
err = duckdb_result_error(&rr);
duckdb_destroy_result(&rr);
return false;
}
duckdb_destroy_result(&rr);
// INSERT rows via VALUES batches (1000 rows/insert).
if (t.rows == 0 || t.cols == 0) return true;
const int batch = 1000;
for (int r0 = 0; r0 < t.rows; r0 += batch) {
int r1 = (r0 + batch < t.rows) ? r0 + batch : t.rows;
std::string ins = "INSERT INTO " + sql_ident(t.name) + " VALUES ";
for (int r = r0; r < r1; ++r) {
if (r > r0) ins += ", ";
ins += "(";
for (int c = 0; c < t.cols; ++c) {
if (c > 0) ins += ", ";
const char* v = t.cells[r * t.cols + c];
if (!v || !*v) { ins += "NULL"; continue; }
ColumnType ct = (c < (int)t.types.size()) ? t.types[c] : ColumnType::String;
if (ct == ColumnType::Int || ct == ColumnType::Float) {
// Asumir parseable; sino DuckDB error.
ins += v;
} else if (ct == ColumnType::Bool) {
ins += (std::strcmp(v, "true") == 0) ? "TRUE" : "FALSE";
} else {
ins += lit_str(v);
}
}
ins += ")";
}
ins += ";";
if (duckdb_query(cn, ins.c_str(), &rr) == DuckDBError) {
err = std::string("INSERT into ") + t.name + ": " + duckdb_result_error(&rr);
duckdb_destroy_result(&rr);
return false;
}
duckdb_destroy_result(&rr);
}
return true;
}
ColumnType type_from_duckdb(duckdb_type t) {
switch (t) {
case DUCKDB_TYPE_BOOLEAN: return ColumnType::Bool;
case DUCKDB_TYPE_TINYINT:
case DUCKDB_TYPE_SMALLINT:
case DUCKDB_TYPE_INTEGER:
case DUCKDB_TYPE_BIGINT:
case DUCKDB_TYPE_HUGEINT:
case DUCKDB_TYPE_UTINYINT:
case DUCKDB_TYPE_USMALLINT:
case DUCKDB_TYPE_UINTEGER:
case DUCKDB_TYPE_UBIGINT:
return ColumnType::Int;
case DUCKDB_TYPE_FLOAT:
case DUCKDB_TYPE_DOUBLE:
case DUCKDB_TYPE_DECIMAL:
return ColumnType::Float;
case DUCKDB_TYPE_DATE:
case DUCKDB_TYPE_TIMESTAMP:
return ColumnType::Date;
default:
return ColumnType::String;
}
}
} // anon
Result execute(const std::string& sql,
const std::vector<std::string>& params,
const std::vector<TableInput>& tables) {
Result out;
auto t0 = std::chrono::steady_clock::now();
duckdb_database db = nullptr;
duckdb_connection cn = nullptr;
if (duckdb_open(nullptr, &db) == DuckDBError) {
out.error = "duckdb_open failed";
return out;
}
if (duckdb_connect(db, &cn) == DuckDBError) {
out.error = "duckdb_connect failed";
duckdb_close(&db);
return out;
}
// Crear y poblar tablas.
for (const auto& t : tables) {
std::string e;
if (!create_and_load(cn, t, e)) {
out.error = e;
duckdb_disconnect(&cn);
duckdb_close(&db);
return out;
}
}
// Preparar + bind params.
duckdb_prepared_statement stmt = nullptr;
if (duckdb_prepare(cn, sql.c_str(), &stmt) == DuckDBError) {
out.error = std::string("prepare: ") + duckdb_prepare_error(stmt);
duckdb_destroy_prepare(&stmt);
duckdb_disconnect(&cn);
duckdb_close(&db);
return out;
}
for (size_t i = 0; i < params.size(); ++i) {
// DuckDB params son 1-based.
if (duckdb_bind_varchar(stmt, (idx_t)(i + 1), params[i].c_str()) == DuckDBError) {
out.error = "bind param fail";
duckdb_destroy_prepare(&stmt);
duckdb_disconnect(&cn);
duckdb_close(&db);
return out;
}
}
duckdb_result res;
if (duckdb_execute_prepared(stmt, &res) == DuckDBError) {
out.error = std::string("execute: ") + duckdb_result_error(&res);
duckdb_destroy_result(&res);
duckdb_destroy_prepare(&stmt);
duckdb_disconnect(&cn);
duckdb_close(&db);
return out;
}
// Materializar resultado en StageOutput.
idx_t cols = duckdb_column_count(&res);
idx_t rows = duckdb_row_count(&res);
out.out.cols = (int)cols;
out.out.rows = (int)rows;
out.row_count = (int)rows;
out.out.headers.reserve(cols);
out.out.types.reserve(cols);
for (idx_t c = 0; c < cols; ++c) {
out.out.headers.emplace_back(duckdb_column_name(&res, c));
out.out.types.push_back(type_from_duckdb(duckdb_column_type(&res, c)));
}
out.out.cell_backing.reserve(rows * cols);
out.out.cells.reserve(rows * cols);
for (idx_t r = 0; r < rows; ++r) {
for (idx_t c = 0; c < cols; ++c) {
char* v = duckdb_value_varchar(&res, c, r);
out.out.cell_backing.emplace_back(v ? v : "");
if (v) duckdb_free(v);
}
}
for (auto& s : out.out.cell_backing) out.out.cells.push_back(s.c_str());
duckdb_destroy_result(&res);
duckdb_destroy_prepare(&stmt);
duckdb_disconnect(&cn);
duckdb_close(&db);
auto t1 = std::chrono::steady_clock::now();
out.duration_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
return out;
}
} // namespace tql_duckdb
#endif // FN_TQL_DUCKDB
cpp/apps/primitives_gallery/playground/tables/tql_duckdb.h
+29
View File
@@ -0,0 +1,29 @@
// tql_duckdb: ejecuta SQL DuckDB sobre TableInputs in-memory.
// Solo se compila si FN_TQL_DUCKDB esta definido. Adapter opcional para
// tql_to_sql emit -> execute. Ver issue 0080.
#pragma once
#ifdef FN_TQL_DUCKDB
#include "data_table_logic.h"
#include <string>
#include <vector>
namespace tql_duckdb {
struct Result {
data_table::StageOutput out;
std::string error; // non-empty si fallo
int row_count = 0;
double duration_ms = 0.0;
};
// Impure: abre DuckDB in-memory, registra tablas como CREATE TABLE + INSERT,
// prepara sql con `?` placeholders bound a `params`, materializa resultado.
Result execute(const std::string& sql,
const std::vector<std::string>& params,
const std::vector<data_table::TableInput>& tables);
} // namespace tql_duckdb
#endif // FN_TQL_DUCKDB