diff --git a/CMakeLists.txt b/CMakeLists.txt index 44dad20..cdf549e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,8 @@ endif() add_imgui_app(odr_console main.cpp data_registry.cpp + data_collectors.cpp + runner.cpp ) target_include_directories(odr_console PRIVATE diff --git a/data_collectors.cpp b/data_collectors.cpp new file mode 100644 index 0000000..7194524 --- /dev/null +++ b/data_collectors.cpp @@ -0,0 +1,77 @@ +#include "data_collectors.h" + +#include +#include + +namespace fs = std::filesystem; + +namespace { + +std::string trim(std::string s) { + auto issp = [](unsigned char c) { return std::isspace(c); }; + while (!s.empty() && issp((unsigned char)s.back())) s.pop_back(); + size_t i = 0; + while (i < s.size() && issp((unsigned char)s[i])) ++i; + return s.substr(i); +} + +// Strip wrapping quotes if present. +std::string unquote(std::string s) { + if (s.size() >= 2) { + char a = s.front(), b = s.back(); + if ((a == '"' && b == '"') || (a == '\'' && b == '\'')) { + return s.substr(1, s.size() - 2); + } + } + return s; +} + +// Parsing minimo: busca lineas top-level `id:`, `name:`, `description:`. +// Suficiente para metadata de collectors. Si la app necesita mas, anadir yaml-cpp. +void parse_manifest(const fs::path& yaml_path, Collector& c) { + std::ifstream f(yaml_path); + if (!f.is_open()) return; + std::string line; + while (std::getline(f, line)) { + std::string t = trim(line); + if (t.empty() || t[0] == '#') continue; + // Ignora lineas indentadas (sub-keys de listas/dicts). + if (line.size() > t.size() && line[0] == ' ') continue; + + auto colon = t.find(':'); + if (colon == std::string::npos) continue; + std::string key = trim(t.substr(0, colon)); + std::string val = trim(t.substr(colon + 1)); + val = unquote(val); + + if (key == "id" && !val.empty()) c.id = val; + else if (key == "name" && !val.empty()) c.name = val; + else if (key == "description" && !val.empty()) c.description = val; + } +} + +} // namespace + +bool collectors_discover(const std::string& collectors_root, + std::vector& out) { + out.clear(); + std::error_code ec; + if (!fs::is_directory(collectors_root, ec)) return false; + + for (auto& entry : fs::directory_iterator(collectors_root, ec)) { + if (ec) break; + if (!entry.is_directory()) continue; + Collector c; + c.dir = entry.path().string(); + c.id = entry.path().filename().string(); + c.run_py = (entry.path() / "run.py").string(); + c.has_run = fs::exists(c.run_py); + + fs::path manifest = entry.path() / "manifest.yaml"; + if (fs::exists(manifest)) parse_manifest(manifest, c); + if (c.name.empty()) c.name = c.id; + + out.push_back(std::move(c)); + } + return true; +} diff --git a/data_collectors.h b/data_collectors.h new file mode 100644 index 0000000..11dc193 --- /dev/null +++ b/data_collectors.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +struct Collector { + std::string id; // basename del directorio (ej. "api_hn_top") + std::string name; // del manifest (fallback id) + std::string description; // del manifest + std::string dir; // path absoluto al directorio del collector + std::string run_py; // path absoluto al run.py + bool has_run = false; +}; + +// Enumera /*/manifest.yaml + run.py. +// Parsing manifest minimo: solo lee `id`, `name:`, `description:` por linea (sin yaml-cpp). +bool collectors_discover(const std::string& collectors_root, + std::vector& out); diff --git a/main.cpp b/main.cpp index 6e9b703..1cfae53 100644 --- a/main.cpp +++ b/main.cpp @@ -12,19 +12,34 @@ #include "core/logger.h" #include "data_registry.h" +#include "data_collectors.h" +#include "runner.h" #include #include #include +#include +#include #include #include static OdrRegistry g_registry; static std::string g_db_path; +static std::string g_app_dir; // dir de odr_console (collectors/, ops db) +static std::string g_python_exe; // resolved al arrancar +static std::string g_registry_root; // FN_REGISTRY_ROOT +static std::string g_ops_db_path; // /operations.db + static char g_search_buf[256] = ""; static std::vector g_results; static int g_selected = -1; +static std::vector g_collectors; +static int g_coll_selected = -1; +static int g_coll_param_limit = 30; +static std::string g_last_run_summary; // texto status del ultimo run +static bool g_running = false; + static void do_search() { g_results.clear(); g_selected = -1; @@ -35,9 +50,126 @@ static void do_search() { } } -static bool g_show_launcher = true; -static bool g_show_jobs = true; -static bool g_show_datasets = true; +static bool g_show_launcher = true; +static bool g_show_collectors = true; +static bool g_show_jobs = true; +static bool g_show_datasets = true; + +static void run_selected_collector() { + if (g_coll_selected < 0 || + g_coll_selected >= (int)g_collectors.size()) return; + const Collector& c = g_collectors[g_coll_selected]; + if (!c.has_run) return; + + std::ostringstream ctx; + ctx << "{" + << "\"ops_db_path\":\"" << g_ops_db_path << "\"," + << "\"app_dir\":\"" << g_app_dir << "\"," + << "\"registry_root\":\"" << g_registry_root << "\"," + << "\"params\":{\"limit\":" << g_coll_param_limit << "}" + << "}"; + + std::string tmp_dir = std::string(fn::local_path("runner_tmp")); + g_running = true; + auto res = odr::run_collector(g_python_exe, c.run_py, tmp_dir, ctx.str()); + g_running = false; + + std::ostringstream s; + s << "exit=" << res.exit_code + << " ms=" << res.duration_ms + << "\nstdout: " << res.stdout_str + << "\nstderr (last):\n"; + // Solo ultimas 10 lineas de stderr para no inundar UI. + std::vector lines; + std::istringstream ss(res.stderr_str); + std::string ln; + while (std::getline(ss, ln)) lines.push_back(ln); + int start = (int)lines.size() - 10; + if (start < 0) start = 0; + for (int i = start; i < (int)lines.size(); ++i) s << lines[i] << "\n"; + g_last_run_summary = s.str(); +} + +static void draw_collectors() { + if (!g_show_collectors) return; + if (!ImGui::Begin(TI_PLAYER_PLAY " Collectors", &g_show_collectors)) { + ImGui::End(); + return; + } + if (ImGui::Button("Refresh")) { + std::string root = g_app_dir + "/collectors"; + collectors_discover(root, g_collectors); + g_coll_selected = -1; + } + ImGui::SameLine(); + ImGui::TextDisabled("%zu collectors", g_collectors.size()); + + if (g_collectors.empty()) { + ImGui::TextDisabled("No collectors found in %s/collectors/", + g_app_dir.c_str()); + ImGui::End(); + return; + } + + if (ImGui::BeginTable("##collectors", 3, + ImGuiTableFlags_RowBg | ImGuiTableFlags_Borders | + ImGuiTableFlags_Resizable)) { + ImGui::TableSetupColumn("ID", ImGuiTableColumnFlags_WidthFixed, 180); + ImGui::TableSetupColumn("Name", ImGuiTableColumnFlags_WidthFixed, 200); + ImGui::TableSetupColumn("Description", ImGuiTableColumnFlags_WidthStretch); + ImGui::TableHeadersRow(); + + for (int i = 0; i < (int)g_collectors.size(); ++i) { + const auto& c = g_collectors[i]; + ImGui::TableNextRow(); + ImGui::TableSetColumnIndex(0); + bool sel = (i == g_coll_selected); + if (ImGui::Selectable(c.id.c_str(), sel, + ImGuiSelectableFlags_SpanAllColumns)) { + g_coll_selected = i; + } + ImGui::TableSetColumnIndex(1); + ImGui::TextUnformatted(c.name.c_str()); + ImGui::TableSetColumnIndex(2); + ImGui::TextUnformatted(c.description.c_str()); + } + ImGui::EndTable(); + } + + if (g_coll_selected >= 0 && + g_coll_selected < (int)g_collectors.size()) { + const auto& c = g_collectors[g_coll_selected]; + ImGui::Separator(); + ImGui::Text("Selected: %s", c.id.c_str()); + ImGui::TextDisabled("%s", c.run_py.c_str()); + + ImGui::PushItemWidth(120); + ImGui::InputInt("limit", &g_coll_param_limit); + ImGui::PopItemWidth(); + if (g_coll_param_limit < 1) g_coll_param_limit = 1; + if (g_coll_param_limit > 500) g_coll_param_limit = 500; + + ImGui::BeginDisabled(g_running || !c.has_run); + if (ImGui::Button(g_running ? "Running..." : (TI_PLAYER_PLAY " Run"))) { + run_selected_collector(); + } + ImGui::EndDisabled(); + if (!c.has_run) { + ImGui::SameLine(); + ImGui::TextDisabled("(missing run.py)"); + } + } + + if (!g_last_run_summary.empty()) { + ImGui::Separator(); + ImGui::TextUnformatted("Last run:"); + ImGui::BeginChild("##run_summary", ImVec2(0, 200), true); + ImGui::TextUnformatted(g_last_run_summary.c_str()); + ImGui::EndChild(); + } + + ImGui::End(); +} static void draw_launcher() { if (!g_show_launcher) return; @@ -130,14 +262,16 @@ static void draw_datasets() { static void render() { static fn_ui::PanelToggle panels[] = { - { "Launcher", nullptr, &g_show_launcher }, - { "Jobs", nullptr, &g_show_jobs }, - { "Datasets", nullptr, &g_show_datasets }, + { "Launcher", nullptr, &g_show_launcher }, + { "Collectors", nullptr, &g_show_collectors }, + { "Jobs", nullptr, &g_show_jobs }, + { "Datasets", nullptr, &g_show_datasets }, }; fn_ui::app_menubar(panels, sizeof(panels) / sizeof(panels[0]), nullptr); draw_launcher(); + draw_collectors(); draw_jobs(); draw_datasets(); } @@ -170,6 +304,44 @@ int main(int argc, char** argv) { registry_list_recent(g_registry, 50, g_results); } + // Resolver registry_root, app_dir, python_exe. + if (const char* root = std::getenv("FN_REGISTRY_ROOT"); root && *root) { + g_registry_root = root; + } else if (!g_db_path.empty()) { + // Asume registry.db esta en raiz del registry. + std::filesystem::path p(g_db_path); + g_registry_root = p.parent_path().string(); + } + + if (const char* d = std::getenv("ODR_APP_DIR"); d && *d) { + g_app_dir = d; + } else if (!g_registry_root.empty()) { + g_app_dir = g_registry_root + + "/projects/online_data_recopilation/apps/odr_console"; + } + g_ops_db_path = g_app_dir + "/operations.db"; + + if (const char* py = std::getenv("FN_PYTHON"); py && *py) { + g_python_exe = py; + } else if (!g_registry_root.empty()) { + g_python_exe = g_registry_root + "/python/.venv/bin/python3"; + } else { + g_python_exe = "python3"; + } + + // Discover collectors: prefer app_dir/collectors, fallback a + // /assets/collectors/ (Windows distribuible). + if (!g_app_dir.empty()) { + collectors_discover(g_app_dir + "/collectors", g_collectors); + } + if (g_collectors.empty()) { + const char* assets = fn::asset_dir(); + if (assets && *assets) { + std::string assets_collectors = std::string(assets) + "/collectors"; + collectors_discover(assets_collectors, g_collectors); + } + } + fn::AppConfig cfg; cfg.title = "odr_console — online data recopilation"; cfg.width = 1400; diff --git a/runner.cpp b/runner.cpp new file mode 100644 index 0000000..6ea4cdd --- /dev/null +++ b/runner.cpp @@ -0,0 +1,124 @@ +#include "runner.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace odr { + +namespace { + +long long now_ms() { + using namespace std::chrono; + return duration_cast( + system_clock::now().time_since_epoch()).count(); +} + +std::string make_uid() { + static std::atomic ctr{0}; + long long ts = now_ms(); + int n = ctr.fetch_add(1, std::memory_order_relaxed); + char buf[64]; + std::snprintf(buf, sizeof(buf), "%lld_%05d", ts, n); + return buf; +} + +bool write_file(const fs::path& p, const std::string& bytes) { + std::ofstream f(p, std::ios::binary | std::ios::trunc); + if (!f.is_open()) return false; + f.write(bytes.data(), (std::streamsize)bytes.size()); + return f.good(); +} + +std::string read_file(const fs::path& p) { + std::ifstream f(p, std::ios::binary); + if (!f.is_open()) return ""; + std::ostringstream ss; + ss << f.rdbuf(); + return ss.str(); +} + +// Quote para shell: encierra en comillas dobles, escapa los " internos. +// Suficiente para paths del registry — no tocamos shell metacharacters +// porque no hay input de usuario directo (paths controlados por la app). +std::string sh_quote(const std::string& s) { + std::string out; + out.reserve(s.size() + 2); + out.push_back('"'); + for (char c : s) { + if (c == '"' || c == '\\' || c == '$' || c == '`') { + out.push_back('\\'); + } + out.push_back(c); + } + out.push_back('"'); + return out; +} + +} // namespace + +RunResult run_collector(const std::string& python_exe, + const std::string& run_py, + const std::string& tmp_dir, + const std::string& ctx_json) { + RunResult r; + + std::error_code ec; + fs::create_directories(tmp_dir, ec); + if (ec) { + r.stderr_str = "create_directories failed: " + ec.message(); + return r; + } + + std::string uid = make_uid(); + fs::path ctx_path = fs::path(tmp_dir) / ("ctx_" + uid + ".json"); + fs::path out_path = fs::path(tmp_dir) / ("out_" + uid + ".json"); + fs::path err_path = fs::path(tmp_dir) / ("err_" + uid + ".log"); + + if (!write_file(ctx_path, ctx_json)) { + r.stderr_str = "write ctx failed"; + return r; + } + + std::string cmd = + sh_quote(python_exe) + " " + sh_quote(run_py) + + " < " + sh_quote(ctx_path.string()) + + " > " + sh_quote(out_path.string()) + + " 2> " + sh_quote(err_path.string()); + + long long t0 = now_ms(); + int rc = std::system(cmd.c_str()); + r.duration_ms = now_ms() - t0; + +#if defined(__unix__) || defined(__APPLE__) + // POSIX: WEXITSTATUS para extraer exit code real. + if (rc != -1 && WIFEXITED(rc)) { + r.exit_code = WEXITSTATUS(rc); + } else { + r.exit_code = rc; + } +#else + r.exit_code = rc; +#endif + + r.stdout_str = read_file(out_path); + r.stderr_str = read_file(err_path); + + // Cleanup salvo si fallo (dejar para debug). + if (r.exit_code == 0) { + std::error_code rmec; + fs::remove(ctx_path, rmec); + fs::remove(out_path, rmec); + fs::remove(err_path, rmec); + } + + return r; +} + +} // namespace odr diff --git a/runner.h b/runner.h new file mode 100644 index 0000000..04d1e37 --- /dev/null +++ b/runner.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace odr { + +struct RunResult { + int exit_code = -1; + long long duration_ms = 0; + std::string stdout_str; + std::string stderr_str; +}; + +// Synchronous subprocess runner. Bloquea el hilo caller hasta que el +// subprocess termina. UI debe llamarlo desde un thread propio o aceptar +// el bloqueo (MVP: aceptar bloqueo). +// +// Layout temp: +// /ctx_.json — escrito por runner +// /out_.json — stdout capturado +// /err_.log — stderr capturado +// +// Usa std::system. python_exe puede ser path absoluto o ejecutable en PATH. +// run_py es el path absoluto al run.py del collector. +// +// Retorna RunResult con exit_code=0 en exito. +RunResult run_collector(const std::string& python_exe, + const std::string& run_py, + const std::string& tmp_dir, + const std::string& ctx_json); + +} // namespace odr