diff --git a/.gitignore b/.gitignore index 19b91767..755e00c7 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,4 @@ Thumbs.db broken_paths.txt imgui.ini +prompts/ diff --git a/apps/dag_engine/.gitignore b/apps/dag_engine/.gitignore new file mode 100644 index 00000000..5e8edee6 --- /dev/null +++ b/apps/dag_engine/.gitignore @@ -0,0 +1,18 @@ +# Build output +dag_engine +*.exe + +# Frontend build +frontend/dist/ +frontend/node_modules/ + +# Go +vendor/ + +# Editor +.idea/ +.vscode/ +*.swp + +# OS +.DS_Store diff --git a/apps/dag_engine/api.go b/apps/dag_engine/api.go new file mode 100644 index 00000000..e112f30e --- /dev/null +++ b/apps/dag_engine/api.go @@ -0,0 +1,47 @@ +package main + +import ( + "io/fs" + "net/http" +) + +// RegisterAPI sets up all HTTP routes on the given mux. +func RegisterAPI(mux *http.ServeMux, executor *Executor, scheduler *Scheduler, frontendFS fs.FS) { + // API routes. + mux.HandleFunc("GET /api/dags", handleListDags(executor)) + mux.HandleFunc("GET /api/dags/{name}", handleGetDag(executor)) + mux.HandleFunc("POST /api/dags/{name}/run", handleRunDag(executor)) + + mux.HandleFunc("GET /api/runs", handleListRuns(executor)) + mux.HandleFunc("GET /api/runs/{id}", handleGetRun(executor)) + + mux.HandleFunc("POST /api/scheduler/start", handleSchedulerStart(scheduler)) + mux.HandleFunc("POST /api/scheduler/stop", handleSchedulerStop(scheduler)) + mux.HandleFunc("GET /api/scheduler/status", handleSchedulerStatus(scheduler)) + + // Frontend SPA fallback. + if frontendFS != nil { + mux.Handle("/", spaHandler(frontendFS)) + } +} + +// spaHandler serves static files from the embedded FS, falling back to index.html +// for unknown paths (SPA client-side routing). +func spaHandler(fsys fs.FS) http.Handler { + fileServer := http.FileServer(http.FS(fsys)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Try to serve the file directly. + path := r.URL.Path + if path == "/" { + path = "index.html" + } else { + path = path[1:] // strip leading / + } + + if _, err := fs.Stat(fsys, path); err != nil { + // File not found — serve index.html for SPA routing. + r.URL.Path = "/" + } + fileServer.ServeHTTP(w, r) + }) +} diff --git a/apps/dag_engine/app.md b/apps/dag_engine/app.md new file mode 100644 index 00000000..c8e50c4e --- /dev/null +++ b/apps/dag_engine/app.md @@ -0,0 +1,86 @@ +--- +name: dag_engine +lang: go +domain: infra +description: "Motor de ejecucion de DAGs con CLI y interfaz web. Reemplaza Dagu con implementacion propia compatible con el formato YAML existente. Almacena historial de ejecuciones en SQLite." +tags: [service, dag, workflow, scheduler, web, cron] +uses_functions: + - dag_parse_go_core + - dag_validate_go_core + - dag_topo_sort_go_core + - dag_resolve_env_go_core + - parse_cron_expr_go_core + - next_cron_time_go_core + - cron_ticker_go_infra + - cron_match_go_core + - process_spawn_go_infra + - process_wait_go_infra + - process_kill_go_infra +uses_types: + - dag_definition_go_core + - dag_step_go_core + - dag_validation_result_go_core + - cron_schedule_go_core + - process_handle_go_infra + - process_result_go_infra + - DagRun_go_infra + - DagStepResult_go_infra +framework: "net/http + vite + react" +entry_point: "main.go" +dir_path: "apps/dag_engine" +--- + +## Arquitectura + +CLI + servidor web en un unico binario: + +``` +dag-engine run # ejecuta un DAG desde terminal +dag-engine list [dir] # lista DAGs con schedule y estado +dag-engine status [dag_name] # historial de ejecuciones +dag-engine validate # valida sin ejecutar +dag-engine server # arranca HTTP + frontend web +``` + +### Backend (Go) + +- `net/http` con `ServeMux` (Go 1.22+ pattern routing) +- SQLite via `go-sqlite3` para historial de runs +- Executor: parse -> validate -> topo_sort -> spawn/wait por nivel -> store +- Scheduler: cron_ticker por cada DAG con schedule + +### Frontend (Vite + React + Mantine) + +- DagList: tabla de DAGs con schedule, tags, ultimo status +- DagDetail: metadata + "Run Now" + historial +- RunDetail: timeline de steps con stdout/stderr expandible + +### Storage + +SQLite `dag_engine.db`: +- `dag_runs`: id, dag_name, status, trigger, started_at, finished_at, error +- `dag_step_results`: id, run_id, step_name, status, exit_code, stdout, stderr, duration_ms + +### Build + +```bash +cd frontend && pnpm install && pnpm build +cd .. && CGO_ENABLED=1 go build -tags fts5 -o dag-engine . +``` + +### Uso + +```bash +# CLI +./dag-engine run ~/dagu/dags/example.yaml +./dag-engine list ~/dagu/dags/ + +# Servidor web +./dag-engine server --port 8090 --dags-dir ~/dagu/dags/ --scheduler +# Browser: http://localhost:8090 +``` + +## Notas + +Compatible con el formato YAML de Dagu. Lee DAGs existentes de `~/dagu/dags/` sin modificaciones. +Puerto por defecto 8090 (mismo que Dagu). diff --git a/apps/dag_engine/config.go b/apps/dag_engine/config.go new file mode 100644 index 00000000..f4936921 --- /dev/null +++ b/apps/dag_engine/config.go @@ -0,0 +1,34 @@ +package main + +import ( + "flag" + "os" + "path/filepath" +) + +// Config holds the runtime configuration for the DAG engine. +type Config struct { + Port int + DagsDir string + DBPath string + AutoScheduler bool +} + +// DefaultConfig returns sensible defaults. +func DefaultConfig() Config { + home, _ := os.UserHomeDir() + return Config{ + Port: 8090, + DagsDir: filepath.Join(home, "dagu", "dags"), + DBPath: "dag_engine.db", + } +} + +// ParseFlags populates config from CLI flags for the "server" subcommand. +func (c *Config) ParseFlags(fs *flag.FlagSet, args []string) error { + fs.IntVar(&c.Port, "port", c.Port, "HTTP server port") + fs.StringVar(&c.DagsDir, "dags-dir", c.DagsDir, "directory containing DAG YAML files") + fs.StringVar(&c.DBPath, "db", c.DBPath, "path to SQLite database") + fs.BoolVar(&c.AutoScheduler, "scheduler", c.AutoScheduler, "auto-start cron scheduler") + return fs.Parse(args) +} diff --git a/apps/dag_engine/executor.go b/apps/dag_engine/executor.go new file mode 100644 index 00000000..44fd9ee9 --- /dev/null +++ b/apps/dag_engine/executor.go @@ -0,0 +1,482 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "fn-registry/functions/core" + "fn-registry/functions/infra" + + "dag-engine/store" +) + +// Executor orchestrates DAG parsing, validation, and execution. +type Executor struct { + store *store.DB + dagsDir string +} + +// NewExecutor creates a new executor. +func NewExecutor(s *store.DB, dagsDir string) *Executor { + return &Executor{store: s, dagsDir: dagsDir} +} + +// ExecuteDAG runs a DAG from a YAML file path and returns the run ID. +// It runs asynchronously: steps execute in topological order with parallel levels. +func (e *Executor) ExecuteDAG(ctx context.Context, dagPath string, trigger string) (string, error) { + data, err := os.ReadFile(dagPath) + if err != nil { + return "", fmt.Errorf("read dag: %w", err) + } + + dag, err := core.DagParse(data) + if err != nil { + return "", fmt.Errorf("parse dag: %w", err) + } + dag.FilePath = dagPath + + // Resolve env variables. + dag = core.DagResolveEnv(dag, os.Environ()) + + // Validate. + result := core.DagValidate(dag) + if !result.Valid { + return "", fmt.Errorf("validate dag: %s", strings.Join(result.Errors, "; ")) + } + + // Create run record. + runID := generateID() + now := time.Now() + run := &store.DagRun{ + ID: runID, + DagName: dag.Name, + DagPath: dagPath, + Status: "running", + Trigger: trigger, + StartedAt: now, + } + if err := e.store.CreateRun(run); err != nil { + return "", fmt.Errorf("create run: %w", err) + } + + // Topological sort. + levels, err := core.DagTopoSort(dag.Steps) + if err != nil { + e.failRun(runID, err) + return runID, err + } + + // Setup DAGU_ENV temp file for inter-step communication. + daguEnvFile, err := os.CreateTemp("", "dagu_env_*") + if err != nil { + e.failRun(runID, err) + return runID, err + } + daguEnvPath := daguEnvFile.Name() + daguEnvFile.Close() + defer os.Remove(daguEnvPath) + + // Track step outputs for ${step_id.stdout} references. + stepOutputs := make(map[string]string) + + // Execute levels. + runFailed := false + var runErr error + + for _, level := range levels { + if runFailed { + // Skip remaining levels, mark steps as skipped. + for _, step := range level { + e.recordStepSkipped(runID, step) + } + continue + } + + var wg sync.WaitGroup + var mu sync.Mutex + levelFailed := false + + for _, step := range level { + step := step + wg.Add(1) + go func() { + defer wg.Done() + + mu.Lock() + if levelFailed { + mu.Unlock() + e.recordStepSkipped(runID, step) + return + } + mu.Unlock() + + err := e.executeStep(ctx, runID, dag, step, daguEnvPath, stepOutputs, &mu) + if err != nil && !step.ContinueOn.Failure { + mu.Lock() + levelFailed = true + runFailed = true + runErr = fmt.Errorf("step %q failed: %w", stepName(step), err) + mu.Unlock() + } + }() + } + wg.Wait() + } + + // Run handlers. + if runFailed { + e.runHandlers(ctx, runID, dag, dag.HandlerOn.Failure, daguEnvPath, stepOutputs) + } else { + e.runHandlers(ctx, runID, dag, dag.HandlerOn.Success, daguEnvPath, stepOutputs) + } + e.runHandlers(ctx, runID, dag, dag.HandlerOn.Exit, daguEnvPath, stepOutputs) + + // Finalize run. + fin := time.Now() + status := "success" + errMsg := "" + if runFailed { + status = "failed" + if runErr != nil { + errMsg = runErr.Error() + } + } + e.store.UpdateRunStatus(runID, status, &fin, errMsg) + + return runID, runErr +} + +// executeStep runs a single step, recording results in the store. +func (e *Executor) executeStep(ctx context.Context, runID string, dag core.DagDefinition, step core.DagStep, daguEnvPath string, outputs map[string]string, mu *sync.Mutex) error { + stepID := generateID() + now := time.Now() + e.store.InsertStepResult(&store.DagStepResult{ + ID: stepID, + RunID: runID, + StepName: stepName(step), + Status: "running", + StartedAt: &now, + }) + + // Build environment. + env := buildStepEnv(dag, step, daguEnvPath, outputs) + + // Determine command. + command := step.Command + if command == "" && step.Script != "" { + command = step.Script + } + if command == "" { + e.store.UpdateStepResult(stepID, "skipped", 0, "", "", nil, 0, "no command or script") + return nil + } + + // Resolve step-level ${VAR} references and ${step_id.stdout} patterns. + mu.Lock() + command = resolveStepRefs(command, outputs) + mu.Unlock() + + // Determine working directory. + dir := step.Dir + if dir == "" { + dir = dag.WorkingDir + } + + shell := step.Shell + if shell == "" { + shell = dag.Shell + } + + // Spawn process. + handle, err := infra.ProcessSpawn(command, dir, env, shell) + if err != nil { + fin := time.Now() + e.store.UpdateStepResult(stepID, "failed", -1, "", "", &fin, time.Since(now).Milliseconds(), err.Error()) + return err + } + + // Wait for process. + result, err := infra.ProcessWait(handle, step.TimeoutSec) + fin := time.Now() + duration := time.Since(now).Milliseconds() + + if err != nil && result.ExitCode == 0 { + result.ExitCode = -1 + } + + status := "success" + errMsg := "" + if result.ExitCode != 0 || err != nil { + status = "failed" + if err != nil { + errMsg = err.Error() + } + } + + e.store.UpdateStepResult(stepID, status, result.ExitCode, result.Stdout, result.Stderr, &fin, duration, errMsg) + + // Store output for ${step_id.stdout} references. + if step.ID != "" || step.Output != "" { + mu.Lock() + key := step.ID + if key == "" { + key = step.Output + } + outputs[key] = strings.TrimSpace(result.Stdout) + mu.Unlock() + } + + // Read DAGU_ENV for inter-step env propagation. + readDaguEnv(daguEnvPath, outputs) + + if status == "failed" { + return fmt.Errorf("exit code %d", result.ExitCode) + } + return nil +} + +func (e *Executor) runHandlers(ctx context.Context, runID string, dag core.DagDefinition, handlers []core.DagStep, daguEnvPath string, outputs map[string]string) { + var mu sync.Mutex + for _, step := range handlers { + e.executeStep(ctx, runID, dag, step, daguEnvPath, outputs, &mu) + } +} + +func (e *Executor) failRun(runID string, err error) { + fin := time.Now() + e.store.UpdateRunStatus(runID, "failed", &fin, err.Error()) +} + +func (e *Executor) recordStepSkipped(runID string, step core.DagStep) { + now := time.Now() + e.store.InsertStepResult(&store.DagStepResult{ + ID: generateID(), + RunID: runID, + StepName: stepName(step), + Status: "skipped", + StartedAt: &now, + }) +} + +// --- helpers --- + +func stepName(s core.DagStep) string { + if s.Name != "" { + return s.Name + } + return s.ID +} + +func buildStepEnv(dag core.DagDefinition, step core.DagStep, daguEnvPath string, outputs map[string]string) []string { + env := os.Environ() + + // Add DAG-level env. + for k, v := range dag.Env { + env = append(env, k+"="+v) + } + + // Add step-level env. + for k, v := range step.Env { + env = append(env, k+"="+v) + } + + // Add DAGU_ENV path. + env = append(env, "DAGU_ENV="+daguEnvPath) + + return env +} + +func resolveStepRefs(command string, outputs map[string]string) string { + for k, v := range outputs { + command = strings.ReplaceAll(command, "${"+k+".stdout}", v) + command = strings.ReplaceAll(command, "$"+k+".stdout", v) + } + return command +} + +func readDaguEnv(path string, outputs map[string]string) { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + outputs[parts[0]] = parts[1] + } + } +} + +// generateID creates a simple time-based unique ID. +func generateID() string { + return fmt.Sprintf("%d-%04x", time.Now().UnixNano(), time.Now().Nanosecond()%0xFFFF) +} + +// --- DAG listing helpers --- + +// DagInfo summarizes a DAG file for listing. +type DagInfo struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Schedule []string `json:"schedule,omitempty"` + Tags []string `json:"tags,omitempty"` + Type string `json:"type,omitempty"` + FilePath string `json:"file_path"` + Valid bool `json:"valid"` + LastRun *store.DagRun `json:"last_run,omitempty"` +} + +// ListDAGs scans a directory for YAML files and returns parsed DAG info. +func (e *Executor) ListDAGs() ([]DagInfo, error) { + entries, err := os.ReadDir(e.dagsDir) + if err != nil { + return nil, fmt.Errorf("read dags dir: %w", err) + } + + var dags []DagInfo + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := filepath.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + path := filepath.Join(e.dagsDir, entry.Name()) + data, err := os.ReadFile(path) + if err != nil { + continue + } + + dag, err := core.DagParse(data) + if err != nil { + dags = append(dags, DagInfo{ + Name: strings.TrimSuffix(entry.Name(), ext), + FilePath: path, + Valid: false, + }) + continue + } + + info := DagInfo{ + Name: dag.Name, + Description: dag.Description, + Schedule: dag.Schedule, + Tags: dag.Tags, + Type: dag.Type, + FilePath: path, + Valid: true, + } + + // Attach last run info. + runs, _, _ := e.store.ListRuns(dag.Name, 1, 0) + if len(runs) > 0 { + info.LastRun = &runs[0] + } + + dags = append(dags, info) + } + + return dags, nil +} + +// GetDAG returns detailed info for a specific DAG by name. +func (e *Executor) GetDAG(name string) (*DagInfo, *core.DagDefinition, *core.DagValidationResult, error) { + // Find the YAML file. + entries, err := os.ReadDir(e.dagsDir) + if err != nil { + return nil, nil, nil, err + } + + for _, entry := range entries { + ext := filepath.Ext(entry.Name()) + base := strings.TrimSuffix(entry.Name(), ext) + if (ext != ".yaml" && ext != ".yml") || base != name { + continue + } + + path := filepath.Join(e.dagsDir, entry.Name()) + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, nil, err + } + + dag, err := core.DagParse(data) + if err != nil { + return nil, nil, nil, fmt.Errorf("parse: %w", err) + } + dag.FilePath = path + + validationResult := core.DagValidate(dag) + + info := &DagInfo{ + Name: dag.Name, + Description: dag.Description, + Schedule: dag.Schedule, + Tags: dag.Tags, + Type: dag.Type, + FilePath: path, + Valid: validationResult.Valid, + } + + runs, _, _ := e.store.ListRuns(dag.Name, 1, 0) + if len(runs) > 0 { + info.LastRun = &runs[0] + } + + return info, &dag, &validationResult, nil + } + + return nil, nil, nil, fmt.Errorf("dag %q not found in %s", name, e.dagsDir) +} + +// ValidateDAG parses and validates a DAG file, printing results. +func ValidateDAG(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + + dag, err := core.DagParse(data) + if err != nil { + return fmt.Errorf("parse error: %w", err) + } + + result := core.DagValidate(dag) + + log.Printf("DAG: %s", dag.Name) + log.Printf("Steps: %d", len(dag.Steps)) + log.Printf("Schedule: %v", dag.Schedule) + + if result.Valid { + log.Printf("Validation: PASS") + log.Printf("Topological levels: %d", len(result.Levels)) + for i, level := range result.Levels { + log.Printf(" Level %d: %v", i, level) + } + } else { + log.Printf("Validation: FAIL") + for _, e := range result.Errors { + log.Printf(" ERROR: %s", e) + } + } + for _, w := range result.Warnings { + log.Printf(" WARNING: %s", w) + } + + if !result.Valid { + return fmt.Errorf("validation failed") + } + return nil +} diff --git a/apps/dag_engine/frontend/index.html b/apps/dag_engine/frontend/index.html new file mode 100644 index 00000000..e551b951 --- /dev/null +++ b/apps/dag_engine/frontend/index.html @@ -0,0 +1,12 @@ + + + + + + DAG Engine + + +
+ + + diff --git a/apps/dag_engine/frontend/package.json b/apps/dag_engine/frontend/package.json new file mode 100644 index 00000000..560e2c31 --- /dev/null +++ b/apps/dag_engine/frontend/package.json @@ -0,0 +1,28 @@ +{ + "name": "dag-engine-frontend", + "private": true, + "version": "1.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@mantine/core": "^9.0.2", + "@mantine/hooks": "^9.0.2", + "@tabler/icons-react": "^3.31.0", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "react-router-dom": "^7.6.1" + }, + "devDependencies": { + "@types/react": "^19.1.6", + "@types/react-dom": "^19.1.6", + "@vitejs/plugin-react": "^4.5.2", + "postcss": "^8.5.4", + "postcss-preset-mantine": "^1.17.0", + "typescript": "~5.8.3", + "vite": "^6.3.5" + } +} diff --git a/apps/dag_engine/frontend/postcss.config.cjs b/apps/dag_engine/frontend/postcss.config.cjs new file mode 100644 index 00000000..bbce5388 --- /dev/null +++ b/apps/dag_engine/frontend/postcss.config.cjs @@ -0,0 +1,5 @@ +module.exports = { + plugins: { + "postcss-preset-mantine": {}, + }, +}; diff --git a/apps/dag_engine/frontend/src/App.tsx b/apps/dag_engine/frontend/src/App.tsx new file mode 100644 index 00000000..024797b7 --- /dev/null +++ b/apps/dag_engine/frontend/src/App.tsx @@ -0,0 +1,32 @@ +import { Routes, Route } from "react-router-dom"; +import { AppShell, Container, Title, Group, Text } from "@mantine/core"; +import { IconTopologyRing } from "@tabler/icons-react"; +import { DagList } from "./pages/DagList"; +import { DagDetail } from "./pages/DagDetail"; +import { RunDetail } from "./pages/RunDetail"; + +export function App() { + return ( + + + + + DAG Engine + + fn_registry workflow executor + + + + + + + + } /> + } /> + } /> + + + + + ); +} diff --git a/apps/dag_engine/frontend/src/api.ts b/apps/dag_engine/frontend/src/api.ts new file mode 100644 index 00000000..8035a5a8 --- /dev/null +++ b/apps/dag_engine/frontend/src/api.ts @@ -0,0 +1,63 @@ +import type { + DagSummary, + DagDetail, + DagRun, + RunDetail, + SchedulerStatus, +} from "./types"; + +const BASE = "/api"; + +async function fetchJSON(path: string, init?: RequestInit): Promise { + const res = await fetch(`${BASE}${path}`, init); + if (!res.ok) { + const err = await res.json().catch(() => ({ error: res.statusText })); + throw new Error(err.error || res.statusText); + } + return res.json(); +} + +export function listDags(): Promise { + return fetchJSON("/dags"); +} + +export function getDag(name: string): Promise { + return fetchJSON(`/dags/${encodeURIComponent(name)}`); +} + +export function triggerDag( + name: string +): Promise<{ status: string; dag: string; message: string }> { + return fetchJSON(`/dags/${encodeURIComponent(name)}/run`, { + method: "POST", + }); +} + +export function listRuns(params?: { + dag?: string; + limit?: number; + offset?: number; +}): Promise<{ runs: DagRun[]; total: number }> { + const search = new URLSearchParams(); + if (params?.dag) search.set("dag", params.dag); + if (params?.limit) search.set("limit", String(params.limit)); + if (params?.offset) search.set("offset", String(params.offset)); + const qs = search.toString(); + return fetchJSON(`/runs${qs ? "?" + qs : ""}`); +} + +export function getRun(id: string): Promise { + return fetchJSON(`/runs/${encodeURIComponent(id)}`); +} + +export function startScheduler(): Promise { + return fetchJSON("/scheduler/start", { method: "POST" }); +} + +export function stopScheduler(): Promise { + return fetchJSON("/scheduler/stop", { method: "POST" }); +} + +export function getSchedulerStatus(): Promise { + return fetchJSON("/scheduler/status"); +} diff --git a/apps/dag_engine/frontend/src/components/StatusBadge.tsx b/apps/dag_engine/frontend/src/components/StatusBadge.tsx new file mode 100644 index 00000000..f8114df5 --- /dev/null +++ b/apps/dag_engine/frontend/src/components/StatusBadge.tsx @@ -0,0 +1,18 @@ +import { Badge } from "@mantine/core"; + +const colorMap: Record = { + success: "green", + failed: "red", + running: "blue", + pending: "gray", + cancelled: "yellow", + skipped: "dimmed", +}; + +export function StatusBadge({ status }: { status: string }) { + return ( + + {status} + + ); +} diff --git a/apps/dag_engine/frontend/src/components/StepTimeline.tsx b/apps/dag_engine/frontend/src/components/StepTimeline.tsx new file mode 100644 index 00000000..10558df1 --- /dev/null +++ b/apps/dag_engine/frontend/src/components/StepTimeline.tsx @@ -0,0 +1,85 @@ +import { Timeline, Text, Code, Collapse, Box, Group } from "@mantine/core"; +import { + IconCircleCheck, + IconCircleX, + IconLoader, + IconCircleMinus, + IconClock, +} from "@tabler/icons-react"; +import { useDisclosure } from "@mantine/hooks"; +import type { DagStepResult } from "../types"; + +const iconMap: Record = { + success: , + failed: , + running: , + skipped: , + pending: , +}; + +function StepItem({ step }: { step: DagStepResult }) { + const [opened, { toggle }] = useDisclosure(step.Status === "failed"); + const hasOutput = step.Stdout || step.Stderr; + + return ( + + + {step.StepName} + + + {step.DurationMs}ms + + {step.ExitCode !== 0 && step.ExitCode !== -1 && ( + + exit {step.ExitCode} + + )} + + } + > + {hasOutput && ( + + + {step.Stdout && ( + + {step.Stdout} + + )} + {step.Stderr && ( + + {step.Stderr} + + )} + + + )} + + ); +} + +export function StepTimeline({ steps }: { steps: DagStepResult[] }) { + const activeIndex = steps.findIndex((s) => s.Status === "running"); + + return ( + = 0 ? activeIndex : steps.length - 1} + bulletSize={24} + > + {steps.map((step) => ( + + ))} + + ); +} diff --git a/apps/dag_engine/frontend/src/main.tsx b/apps/dag_engine/frontend/src/main.tsx new file mode 100644 index 00000000..530eb513 --- /dev/null +++ b/apps/dag_engine/frontend/src/main.tsx @@ -0,0 +1,18 @@ +import "@mantine/core/styles.css"; +import { MantineProvider, createTheme } from "@mantine/core"; +import { createRoot } from "react-dom/client"; +import { BrowserRouter } from "react-router-dom"; +import { App } from "./App"; + +const theme = createTheme({ + primaryColor: "blue", + fontFamily: "system-ui, -apple-system, sans-serif", +}); + +createRoot(document.getElementById("root")!).render( + + + + + +); diff --git a/apps/dag_engine/frontend/src/pages/DagDetail.tsx b/apps/dag_engine/frontend/src/pages/DagDetail.tsx new file mode 100644 index 00000000..d4e9d9f6 --- /dev/null +++ b/apps/dag_engine/frontend/src/pages/DagDetail.tsx @@ -0,0 +1,204 @@ +import { useEffect, useState } from "react"; +import { useParams, useNavigate } from "react-router-dom"; +import { + Title, + Text, + Group, + Button, + Badge, + Stack, + Paper, + Table, + Alert, + Loader, + Code, +} from "@mantine/core"; +import { IconPlayerPlay, IconArrowLeft } from "@tabler/icons-react"; +import { getDag, triggerDag } from "../api"; +import { StatusBadge } from "../components/StatusBadge"; +import type { DagDetail as DagDetailType } from "../types"; + +export function DagDetail() { + const { name } = useParams<{ name: string }>(); + const navigate = useNavigate(); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [triggering, setTriggering] = useState(false); + + const load = async () => { + if (!name) return; + setLoading(true); + try { + setData(await getDag(name)); + setError(null); + } catch (e) { + setError((e as Error).message); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + load(); + }, [name]); + + const handleRun = async () => { + if (!name) return; + setTriggering(true); + try { + await triggerDag(name); + setTimeout(load, 1000); + } catch (e) { + setError((e as Error).message); + } finally { + setTriggering(false); + } + }; + + if (loading) return ; + if (error) return {error}; + if (!data) return Not found; + + const { dag, validation, runs } = data; + + return ( + + + + + + +
+ {dag.Name} + {dag.Description && ( + + {dag.Description} + + )} +
+ + + + + {dag.Schedule?.map((s: string) => ( + + {s} + + ))} + {dag.Type || "chain"} + {dag.Tags?.map((t: string) => ( + + {t} + + ))} + + + {!validation.Valid && ( + + {validation.Errors.map((e: string, i: number) => ( + + {e} + + ))} + + )} + + + + Steps ({dag.Steps?.length || 0}) + + {validation.Levels?.map((level: string[], i: number) => ( + + + Level {i}: + + {level.map((name: string) => { + const step = dag.Steps?.find( + (s) => s.Name === name || s.ID === name + ); + return ( + + {name} + {step?.Depends?.length + ? ` (after ${step.Depends.join(",")})` + : ""} + + ); + })} + + ))} + + {dag.Env && Object.keys(dag.Env).length > 0 && ( + <> + + Environment + + + {Object.entries(dag.Env) + .map(([k, v]) => `${k}=${v}`) + .join("\n")} + + + )} + + + + + Run History + + {runs?.length ? ( + + + + Status + Trigger + Started + Duration + + + + {runs.map((r) => ( + navigate(`/runs/${r.ID}`)} + > + + + + {r.Trigger} + + {new Date(r.StartedAt).toLocaleString()} + + + {r.FinishedAt + ? `${Math.round((new Date(r.FinishedAt).getTime() - new Date(r.StartedAt).getTime()) / 1000)}s` + : "running..."} + + + ))} + +
+ ) : ( + + No runs yet + + )} + + + ); +} diff --git a/apps/dag_engine/frontend/src/pages/DagList.tsx b/apps/dag_engine/frontend/src/pages/DagList.tsx new file mode 100644 index 00000000..7993bd08 --- /dev/null +++ b/apps/dag_engine/frontend/src/pages/DagList.tsx @@ -0,0 +1,164 @@ +import { useEffect, useState } from "react"; +import { useNavigate } from "react-router-dom"; +import { + Table, + Title, + Group, + Button, + Badge, + Text, + Loader, + Stack, + Alert, +} from "@mantine/core"; +import { + IconPlayerPlay, + IconPlayerStop, + IconRefresh, +} from "@tabler/icons-react"; +import { listDags, getSchedulerStatus, startScheduler, stopScheduler } from "../api"; +import { StatusBadge } from "../components/StatusBadge"; +import type { DagSummary, SchedulerStatus } from "../types"; + +export function DagList() { + const [dags, setDags] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [scheduler, setScheduler] = useState(null); + const navigate = useNavigate(); + + const load = async () => { + setLoading(true); + setError(null); + try { + const [d, s] = await Promise.all([listDags(), getSchedulerStatus()]); + setDags(d || []); + setScheduler(s); + } catch (e) { + setError((e as Error).message); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + load(); + const interval = setInterval(load, 10000); + return () => clearInterval(interval); + }, []); + + const toggleScheduler = async () => { + if (scheduler?.running) { + await stopScheduler(); + } else { + await startScheduler(); + } + const s = await getSchedulerStatus(); + setScheduler(s); + }; + + return ( + + + DAGs + + + + + + + {error && {error}} + + {loading && !dags.length ? ( + + ) : ( + + + + Name + Schedule + Type + Tags + Last Status + Last Run + + + + {dags.map((d) => ( + navigate(`/dags/${d.name}`)} + > + + {d.name} + {d.description && ( + + {d.description} + + )} + + + + {d.schedule?.join(", ") || "-"} + + + + + {d.type || "chain"} + + + + + {d.tags?.map((t) => ( + + {t} + + ))} + + + + {d.last_run ? ( + + ) : ( + + - + + )} + + + + {d.last_run + ? new Date(d.last_run.StartedAt).toLocaleString() + : "-"} + + + + ))} + +
+ )} + + ); +} diff --git a/apps/dag_engine/frontend/src/pages/RunDetail.tsx b/apps/dag_engine/frontend/src/pages/RunDetail.tsx new file mode 100644 index 00000000..99e2d80d --- /dev/null +++ b/apps/dag_engine/frontend/src/pages/RunDetail.tsx @@ -0,0 +1,105 @@ +import { useEffect, useState } from "react"; +import { useParams, useNavigate } from "react-router-dom"; +import { + Title, + Text, + Group, + Button, + Stack, + Paper, + Alert, + Loader, +} from "@mantine/core"; +import { IconArrowLeft } from "@tabler/icons-react"; +import { getRun } from "../api"; +import { StatusBadge } from "../components/StatusBadge"; +import { StepTimeline } from "../components/StepTimeline"; +import type { RunDetail as RunDetailType } from "../types"; + +export function RunDetail() { + const { id } = useParams<{ id: string }>(); + const navigate = useNavigate(); + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + const load = async () => { + if (!id) return; + try { + setData(await getRun(id)); + setError(null); + } catch (e) { + setError((e as Error).message); + } finally { + setLoading(false); + } + }; + + useEffect(() => { + load(); + // Auto-refresh while running. + const interval = setInterval(() => { + if (data?.run.Status === "running") { + load(); + } + }, 2000); + return () => clearInterval(interval); + }, [id, data?.run.Status]); + + if (loading) return ; + if (error) return {error}; + if (!data) return Not found; + + const { run, steps } = data; + const duration = run.FinishedAt + ? `${Math.round((new Date(run.FinishedAt).getTime() - new Date(run.StartedAt).getTime()) / 1000)}s` + : "running..."; + + return ( + + + + + + +
+ Run {run.ID.substring(0, 16)}... + + {run.DagName} · {run.Trigger} ·{" "} + {new Date(run.StartedAt).toLocaleString()} + +
+ + + {duration} + + + + {run.Error && ( + + {run.Error} + + )} + + + + Steps ({steps?.length || 0}) + + {steps?.length ? ( + + ) : ( + + No steps recorded + + )} + + + ); +} diff --git a/apps/dag_engine/frontend/src/types.ts b/apps/dag_engine/frontend/src/types.ts new file mode 100644 index 00000000..4c510709 --- /dev/null +++ b/apps/dag_engine/frontend/src/types.ts @@ -0,0 +1,66 @@ +export interface DagSummary { + name: string; + description?: string; + schedule?: string[]; + tags?: string[]; + type?: string; + file_path: string; + valid: boolean; + last_run?: DagRun; +} + +export interface DagRun { + ID: string; + DagName: string; + DagPath: string; + Status: string; + Trigger: string; + StartedAt: string; + FinishedAt?: string; + Error: string; +} + +export interface DagStepResult { + ID: string; + RunID: string; + StepName: string; + Status: string; + ExitCode: number; + Stdout: string; + Stderr: string; + StartedAt?: string; + FinishedAt?: string; + DurationMs: number; + Error: string; +} + +export interface DagDetail { + info: DagSummary; + dag: { + Name: string; + Description: string; + Type: string; + Schedule: string[]; + Steps: { Name: string; ID: string; Command: string; Script: string; Depends: string[] }[]; + Env: Record; + Tags: string[]; + HandlerOn: { Failure: unknown[]; Success: unknown[] }; + }; + validation: { + Valid: boolean; + Errors: string[]; + Warnings: string[]; + Levels: string[][]; + }; + runs: DagRun[]; +} + +export interface RunDetail { + run: DagRun; + steps: DagStepResult[]; +} + +export interface SchedulerStatus { + running: boolean; + dags: { name: string; path: string; schedule: string; next_run: string }[]; +} diff --git a/apps/dag_engine/frontend/tsconfig.json b/apps/dag_engine/frontend/tsconfig.json new file mode 100644 index 00000000..39a405b9 --- /dev/null +++ b/apps/dag_engine/frontend/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "isolatedModules": true, + "moduleDetection": "force", + "noEmit": true, + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + }, + "include": ["src"] +} diff --git a/apps/dag_engine/frontend/vite.config.ts b/apps/dag_engine/frontend/vite.config.ts new file mode 100644 index 00000000..bd5bbb7c --- /dev/null +++ b/apps/dag_engine/frontend/vite.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +export default defineConfig({ + plugins: [react()], + server: { + port: 5175, + proxy: { + "/api": "http://localhost:8090", + }, + }, + build: { + outDir: "dist", + }, +}); diff --git a/apps/dag_engine/go.mod b/apps/dag_engine/go.mod new file mode 100644 index 00000000..5eeabd69 --- /dev/null +++ b/apps/dag_engine/go.mod @@ -0,0 +1,48 @@ +module dag-engine + +go 1.25.0 + +require ( + fn-registry v0.0.0-00010101000000-000000000000 + github.com/mattn/go-sqlite3 v1.14.37 +) + +require ( + github.com/ClickHouse/ch-go v0.71.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.44.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/apache/arrow-go/v18 v18.1.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.1.24+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/marcboeker/go-duckdb v1.8.5 // indirect + github.com/paulmach/orb v0.12.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect + golang.org/x/mod v0.27.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/tools v0.36.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace fn-registry => /home/lucas/fn_registry diff --git a/apps/dag_engine/go.sum b/apps/dag_engine/go.sum new file mode 100644 index 00000000..d8b66118 --- /dev/null +++ b/apps/dag_engine/go.sum @@ -0,0 +1,168 @@ +github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= +github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= +github.com/ClickHouse/clickhouse-go/v2 v2.44.0 h1:9pxs5pRwIvhni5BDRPn/n5A8DeUod5TnBaeulFBX8EQ= +github.com/ClickHouse/clickhouse-go/v2 v2.44.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y= +github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o= +github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0= +github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8= +github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg= +github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc= +golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= +gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/dag_engine/handlers_dags.go b/apps/dag_engine/handlers_dags.go new file mode 100644 index 00000000..8998d6de --- /dev/null +++ b/apps/dag_engine/handlers_dags.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" +) + +func handleListDags(executor *Executor) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + dags, err := executor.ListDAGs() + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, dags) + } +} + +func handleGetDag(executor *Executor) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + info, dag, validation, err := executor.GetDAG(name) + if err != nil { + writeError(w, http.StatusNotFound, err.Error()) + return + } + + // Get recent runs. + runs, _, _ := executor.store.ListRuns(dag.Name, 10, 0) + + resp := map[string]interface{}{ + "info": info, + "dag": dag, + "validation": validation, + "runs": runs, + } + writeJSON(w, http.StatusOK, resp) + } +} + +func handleRunDag(executor *Executor) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + info, _, _, err := executor.GetDAG(name) + if err != nil { + writeError(w, http.StatusNotFound, err.Error()) + return + } + + // Execute asynchronously. + go func() { + ctx := context.Background() + executor.ExecuteDAG(ctx, info.FilePath, "api") + }() + + // Return run acknowledgment. + writeJSON(w, http.StatusAccepted, map[string]string{ + "status": "accepted", + "dag": name, + "message": "DAG execution started", + }) + } +} + +// --- JSON helpers --- + +func writeJSON(w http.ResponseWriter, status int, data interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(data) +} + +func writeError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, map[string]string{"error": msg}) +} diff --git a/apps/dag_engine/handlers_runs.go b/apps/dag_engine/handlers_runs.go new file mode 100644 index 00000000..cdd0811c --- /dev/null +++ b/apps/dag_engine/handlers_runs.go @@ -0,0 +1,59 @@ +package main + +import ( + "net/http" + "strconv" +) + +func handleListRuns(executor *Executor) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + dagName := r.URL.Query().Get("dag") + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) + if limit <= 0 || limit > 100 { + limit = 20 + } + if offset < 0 { + offset = 0 + } + + runs, total, err := executor.store.ListRuns(dagName, limit, offset) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + writeJSON(w, http.StatusOK, map[string]interface{}{ + "runs": runs, + "total": total, + "limit": limit, + "offset": offset, + }) + } +} + +func handleGetRun(executor *Executor) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + run, err := executor.store.GetRun(id) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if run == nil { + writeError(w, http.StatusNotFound, "run not found") + return + } + + steps, err := executor.store.ListStepResults(id) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + writeJSON(w, http.StatusOK, map[string]interface{}{ + "run": run, + "steps": steps, + }) + } +} diff --git a/apps/dag_engine/handlers_scheduler.go b/apps/dag_engine/handlers_scheduler.go new file mode 100644 index 00000000..c7e33abf --- /dev/null +++ b/apps/dag_engine/handlers_scheduler.go @@ -0,0 +1,27 @@ +package main + +import "net/http" + +func handleSchedulerStart(scheduler *Scheduler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := scheduler.Start(); err != nil { + writeError(w, http.StatusConflict, err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "started"}) + } +} + +func handleSchedulerStop(scheduler *Scheduler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + scheduler.Stop() + writeJSON(w, http.StatusOK, map[string]string{"status": "stopped"}) + } +} + +func handleSchedulerStatus(scheduler *Scheduler) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + status := scheduler.Status() + writeJSON(w, http.StatusOK, status) + } +} diff --git a/apps/dag_engine/main.go b/apps/dag_engine/main.go new file mode 100644 index 00000000..a3498001 --- /dev/null +++ b/apps/dag_engine/main.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "embed" + "flag" + "fmt" + iofs "io/fs" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "text/tabwriter" + "time" + + "fn-registry/functions/core" + + "dag-engine/store" +) + +//go:embed all:frontend/dist +var frontendDist embed.FS + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + cmd := os.Args[1] + args := os.Args[2:] + + switch cmd { + case "run": + cmdRun(args) + case "list": + cmdList(args) + case "status": + cmdStatus(args) + case "validate": + cmdValidate(args) + case "server": + cmdServer(args) + case "help", "-h", "--help": + printUsage() + default: + fmt.Fprintf(os.Stderr, "unknown command: %s\n", cmd) + printUsage() + os.Exit(1) + } +} + +func printUsage() { + fmt.Println(`dag-engine — DAG workflow executor + +Usage: + dag-engine [options] + +Commands: + run Execute a DAG and show results + list [dir] List DAGs with schedule and last status + status [dag_name] Show execution history + validate Parse and validate without executing + server Start HTTP server with web frontend + +Server options: + --port HTTP port (default: 8090) + --dags-dir DAGs directory (default: ~/dagu/dags) + --db SQLite database path (default: dag_engine.db) + --scheduler Auto-start cron scheduler`) +} + +// --- CLI Commands --- + +func cmdRun(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: dag-engine run ") + os.Exit(1) + } + + dagPath := args[0] + cfg := DefaultConfig() + + // Parse optional flags after the path. + fs := flag.NewFlagSet("run", flag.ExitOnError) + fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path") + fs.Parse(args[1:]) + + db, err := store.Open(cfg.DBPath) + if err != nil { + log.Fatalf("open db: %v", err) + } + defer db.Close() + + executor := NewExecutor(db, filepath.Dir(dagPath)) + + fmt.Printf("Executing %s...\n", dagPath) + ctx := context.Background() + runID, err := executor.ExecuteDAG(ctx, dagPath, "manual") + + // Print results. + if runID != "" { + run, _ := db.GetRun(runID) + steps, _ := db.ListStepResults(runID) + + if run != nil { + fmt.Println() + for _, s := range steps { + icon := " " + switch s.Status { + case "success": + icon = "OK" + case "failed": + icon = "!!" + case "skipped": + icon = "--" + case "running": + icon = ".." + } + fmt.Printf("[%s] %s (%dms)\n", icon, s.StepName, s.DurationMs) + if s.Status == "failed" && s.Stderr != "" { + for _, line := range strings.Split(strings.TrimSpace(s.Stderr), "\n") { + fmt.Printf(" %s\n", line) + } + } + } + fmt.Println() + + dur := "" + if run.FinishedAt != nil { + dur = fmt.Sprintf(" (%s)", run.FinishedAt.Sub(run.StartedAt).Round(time.Millisecond)) + } + fmt.Printf("Run %s: %s%s\n", runID, strings.ToUpper(run.Status), dur) + } + } + + if err != nil { + os.Exit(1) + } +} + +func cmdList(args []string) { + cfg := DefaultConfig() + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + cfg.DagsDir = args[0] + args = args[1:] + } + + fs := flag.NewFlagSet("list", flag.ExitOnError) + fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path") + fs.Parse(args) + + db, err := store.Open(cfg.DBPath) + if err != nil { + log.Fatalf("open db: %v", err) + } + defer db.Close() + + executor := NewExecutor(db, cfg.DagsDir) + dags, err := executor.ListDAGs() + if err != nil { + log.Fatalf("list dags: %v", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "NAME\tSCHEDULE\tTYPE\tTAGS\tLAST STATUS\tLAST RUN") + for _, d := range dags { + sched := strings.Join(d.Schedule, ", ") + tags := strings.Join(d.Tags, ", ") + lastStatus := "-" + lastRun := "-" + if d.LastRun != nil { + lastStatus = d.LastRun.Status + lastRun = d.LastRun.StartedAt.Format("2006-01-02 15:04") + } + typ := d.Type + if typ == "" { + typ = "chain" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", d.Name, sched, typ, tags, lastStatus, lastRun) + } + w.Flush() +} + +func cmdStatus(args []string) { + cfg := DefaultConfig() + + fs := flag.NewFlagSet("status", flag.ExitOnError) + fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path") + limit := fs.Int("limit", 10, "number of runs to show") + fs.Parse(args) + + dagName := "" + if fs.NArg() > 0 { + dagName = fs.Arg(0) + } + + db, err := store.Open(cfg.DBPath) + if err != nil { + log.Fatalf("open db: %v", err) + } + defer db.Close() + + runs, total, err := db.ListRuns(dagName, *limit, 0) + if err != nil { + log.Fatalf("list runs: %v", err) + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "Showing %d of %d runs", len(runs), total) + if dagName != "" { + fmt.Fprintf(w, " for %s", dagName) + } + fmt.Fprintln(w) + fmt.Fprintln(w, "RUN_ID\tDAG\tSTATUS\tTRIGGER\tSTARTED\tDURATION") + for _, r := range runs { + dur := "-" + if r.FinishedAt != nil { + dur = r.FinishedAt.Sub(r.StartedAt).Round(time.Millisecond).String() + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", + r.ID, r.DagName, r.Status, r.Trigger, + r.StartedAt.Format("2006-01-02 15:04:05"), dur) + } + w.Flush() +} + +func cmdValidate(args []string) { + if len(args) < 1 { + fmt.Fprintln(os.Stderr, "usage: dag-engine validate ") + os.Exit(1) + } + + data, err := os.ReadFile(args[0]) + if err != nil { + log.Fatalf("read: %v", err) + } + + dag, err := core.DagParse(data) + if err != nil { + log.Fatalf("parse error: %v", err) + } + + result := core.DagValidate(dag) + + fmt.Printf("DAG: %s\n", dag.Name) + fmt.Printf("Steps: %d\n", len(dag.Steps)) + fmt.Printf("Schedule: %v\n", dag.Schedule) + fmt.Printf("Type: %s\n", dag.Type) + + if result.Valid { + fmt.Println("Validation: PASS") + for i, level := range result.Levels { + fmt.Printf(" Level %d: %v\n", i, level) + } + } else { + fmt.Println("Validation: FAIL") + for _, e := range result.Errors { + fmt.Printf(" ERROR: %s\n", e) + } + } + for _, w := range result.Warnings { + fmt.Printf(" WARNING: %s\n", w) + } + + if !result.Valid { + os.Exit(1) + } +} + +// --- Server Command --- + +func cmdServer(args []string) { + cfg := DefaultConfig() + fs := flag.NewFlagSet("server", flag.ExitOnError) + cfg.ParseFlags(fs, args) + + db, err := store.Open(cfg.DBPath) + if err != nil { + log.Fatalf("open db: %v", err) + } + defer db.Close() + + executor := NewExecutor(db, cfg.DagsDir) + scheduler := NewScheduler(executor, cfg.DagsDir) + + // Prepare frontend FS. + var feFS iofs.FS + distFS, err := iofs.Sub(frontendDist, "frontend/dist") + if err == nil { + // Check if dist has content (built frontend exists). + entries, _ := iofs.ReadDir(distFS, ".") + if len(entries) > 0 { + feFS = distFS + log.Printf("serving frontend from embedded dist/") + } + } + if feFS == nil { + log.Printf("no frontend build found, API-only mode") + } + + mux := http.NewServeMux() + RegisterAPI(mux, executor, scheduler, feFS) + + handler := corsMiddleware(loggingMiddleware(mux)) + + if cfg.AutoScheduler { + if err := scheduler.Start(); err != nil { + log.Printf("scheduler start: %v", err) + } + } + + addr := fmt.Sprintf(":%d", cfg.Port) + log.Printf("dag-engine server starting on http://0.0.0.0%s", addr) + log.Printf("dags dir: %s", cfg.DagsDir) + log.Printf("database: %s", cfg.DBPath) + + srv := &http.Server{Addr: addr, Handler: handler} + + // Graceful shutdown. + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + log.Println("shutting down...") + scheduler.Stop() + srv.Shutdown(context.Background()) + }() + + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + log.Fatalf("server: %v", err) + } +} diff --git a/apps/dag_engine/middleware.go b/apps/dag_engine/middleware.go new file mode 100644 index 00000000..5e42d955 --- /dev/null +++ b/apps/dag_engine/middleware.go @@ -0,0 +1,30 @@ +package main + +import ( + "log" + "net/http" + "time" +) + +// corsMiddleware adds permissive CORS headers for development. +func corsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + next.ServeHTTP(w, r) + }) +} + +// loggingMiddleware logs each HTTP request with method, path and duration. +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s", r.Method, r.URL.Path, time.Since(start).Round(time.Millisecond)) + }) +} diff --git a/apps/dag_engine/scheduler.go b/apps/dag_engine/scheduler.go new file mode 100644 index 00000000..a2da6d8e --- /dev/null +++ b/apps/dag_engine/scheduler.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "fn-registry/functions/core" + "fn-registry/functions/infra" +) + +// ScheduledDAG represents a DAG with a parsed cron schedule. +type ScheduledDAG struct { + Name string `json:"name"` + Path string `json:"path"` + Schedule string `json:"schedule"` + NextRun time.Time `json:"next_run"` +} + +// Scheduler manages cron-triggered DAG execution. +type Scheduler struct { + mu sync.Mutex + running bool + cancel context.CancelFunc + dagsDir string + executor *Executor + dags []ScheduledDAG +} + +// NewScheduler creates a new scheduler. +func NewScheduler(executor *Executor, dagsDir string) *Scheduler { + return &Scheduler{ + executor: executor, + dagsDir: dagsDir, + } +} + +// Start scans for DAGs with schedules and starts cron tickers for each. +func (s *Scheduler) Start() error { + s.mu.Lock() + if s.running { + s.mu.Unlock() + return fmt.Errorf("scheduler already running") + } + + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + s.running = true + s.mu.Unlock() + + scheduled, err := s.scanDAGs() + if err != nil { + s.mu.Lock() + s.running = false + s.mu.Unlock() + cancel() + return err + } + + s.mu.Lock() + s.dags = scheduled + s.mu.Unlock() + + log.Printf("[scheduler] started with %d DAGs", len(scheduled)) + + for _, dag := range scheduled { + dag := dag + go s.runTicker(ctx, dag) + } + + return nil +} + +// Stop cancels all tickers and stops the scheduler. +func (s *Scheduler) Stop() { + s.mu.Lock() + defer s.mu.Unlock() + if !s.running { + return + } + s.cancel() + s.running = false + s.dags = nil + log.Printf("[scheduler] stopped") +} + +// IsRunning returns true if the scheduler is active. +func (s *Scheduler) IsRunning() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.running +} + +// Status returns the list of scheduled DAGs with their next run time. +type SchedulerStatus struct { + Running bool `json:"running"` + DAGs []ScheduledDAG `json:"dags"` +} + +func (s *Scheduler) Status() SchedulerStatus { + s.mu.Lock() + defer s.mu.Unlock() + return SchedulerStatus{ + Running: s.running, + DAGs: s.dags, + } +} + +// scanDAGs reads the dags directory and returns DAGs that have cron schedules. +func (s *Scheduler) scanDAGs() ([]ScheduledDAG, error) { + entries, err := os.ReadDir(s.dagsDir) + if err != nil { + return nil, err + } + + var scheduled []ScheduledDAG + for _, entry := range entries { + ext := filepath.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + path := filepath.Join(s.dagsDir, entry.Name()) + data, err := os.ReadFile(path) + if err != nil { + continue + } + + dag, err := core.DagParse(data) + if err != nil { + continue + } + + for _, expr := range dag.Schedule { + sched, err := core.ParseCronExpr(strings.TrimSpace(expr)) + if err != nil { + log.Printf("[scheduler] invalid cron %q in %s: %v", expr, dag.Name, err) + continue + } + next := core.NextCronTime(sched, time.Now()) + scheduled = append(scheduled, ScheduledDAG{ + Name: dag.Name, + Path: path, + Schedule: expr, + NextRun: next, + }) + } + } + + return scheduled, nil +} + +// runTicker starts a cron ticker for a single DAG schedule. +func (s *Scheduler) runTicker(ctx context.Context, dag ScheduledDAG) { + sched, err := core.ParseCronExpr(strings.TrimSpace(dag.Schedule)) + if err != nil { + return + } + + // Convert core.CronSchedule to infra.CronTickerSchedule. + tickerSched := infra.CronTickerSchedule{ + Minute: sched.Minute, + Hour: sched.Hour, + DayOfMonth: sched.DayOfMonth, + Month: sched.Month, + DayOfWeek: sched.DayOfWeek, + } + + ch := infra.CronTicker(tickerSched, ctx) + log.Printf("[scheduler] ticker started for %s (%s), next: %s", dag.Name, dag.Schedule, dag.NextRun.Format(time.RFC3339)) + + for t := range ch { + log.Printf("[scheduler] triggered %s at %s", dag.Name, t.Format(time.RFC3339)) + go func() { + runID, err := s.executor.ExecuteDAG(ctx, dag.Path, "cron") + if err != nil { + log.Printf("[scheduler] %s failed: %v (run: %s)", dag.Name, err, runID) + } else { + log.Printf("[scheduler] %s completed (run: %s)", dag.Name, runID) + } + }() + } +} diff --git a/apps/dag_engine/store/migrations/001_init.sql b/apps/dag_engine/store/migrations/001_init.sql new file mode 100644 index 00000000..218ae39c --- /dev/null +++ b/apps/dag_engine/store/migrations/001_init.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS dag_runs ( + id TEXT PRIMARY KEY, + dag_name TEXT NOT NULL, + dag_path TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','success','failed','cancelled')), + trigger TEXT NOT NULL DEFAULT 'manual' CHECK(trigger IN ('manual','cron','api')), + started_at TEXT NOT NULL, + finished_at TEXT, + error TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS dag_step_results ( + id TEXT PRIMARY KEY, + run_id TEXT NOT NULL REFERENCES dag_runs(id) ON DELETE CASCADE, + step_name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','success','failed','skipped')), + exit_code INTEGER NOT NULL DEFAULT -1, + stdout TEXT NOT NULL DEFAULT '', + stderr TEXT NOT NULL DEFAULT '', + started_at TEXT, + finished_at TEXT, + duration_ms INTEGER NOT NULL DEFAULT 0, + error TEXT NOT NULL DEFAULT '' +); + +CREATE INDEX IF NOT EXISTS idx_runs_dag_name ON dag_runs(dag_name); +CREATE INDEX IF NOT EXISTS idx_runs_status ON dag_runs(status); +CREATE INDEX IF NOT EXISTS idx_runs_started ON dag_runs(started_at DESC); +CREATE INDEX IF NOT EXISTS idx_step_results_run ON dag_step_results(run_id); diff --git a/apps/dag_engine/store/store.go b/apps/dag_engine/store/store.go new file mode 100644 index 00000000..9f049062 --- /dev/null +++ b/apps/dag_engine/store/store.go @@ -0,0 +1,231 @@ +package store + +import ( + "database/sql" + _ "embed" + "fmt" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +//go:embed migrations/001_init.sql +var migrationSQL string + +// DB wraps a SQLite connection for DAG run persistence. +type DB struct { + conn *sql.DB + path string +} + +// Open opens or creates a DAG engine database at the given path. +func Open(path string) (*DB, error) { + conn, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on") + if err != nil { + return nil, fmt.Errorf("store: open %s: %w", path, err) + } + if _, err := conn.Exec(migrationSQL); err != nil { + conn.Close() + return nil, fmt.Errorf("store: migrate: %w", err) + } + return &DB{conn: conn, path: path}, nil +} + +// Close closes the database connection. +func (db *DB) Close() error { + return db.conn.Close() +} + +// --- DagRun CRUD --- + +// DagRun mirrors infra.DagRun for the store layer. +type DagRun struct { + ID string + DagName string + DagPath string + Status string + Trigger string + StartedAt time.Time + FinishedAt *time.Time + Error string +} + +// CreateRun inserts a new run record. +func (db *DB) CreateRun(run *DagRun) error { + _, err := db.conn.Exec( + `INSERT INTO dag_runs (id, dag_name, dag_path, status, trigger, started_at, error) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + run.ID, run.DagName, run.DagPath, run.Status, run.Trigger, + run.StartedAt.Format(time.RFC3339), run.Error, + ) + return err +} + +// UpdateRunStatus updates a run's status and optionally its finished_at and error. +func (db *DB) UpdateRunStatus(id, status string, finishedAt *time.Time, errMsg string) error { + var fin *string + if finishedAt != nil { + s := finishedAt.Format(time.RFC3339) + fin = &s + } + _, err := db.conn.Exec( + `UPDATE dag_runs SET status=?, finished_at=?, error=? WHERE id=?`, + status, fin, errMsg, id, + ) + return err +} + +// GetRun retrieves a single run by ID. +func (db *DB) GetRun(id string) (*DagRun, error) { + row := db.conn.QueryRow( + `SELECT id, dag_name, dag_path, status, trigger, started_at, finished_at, error + FROM dag_runs WHERE id=?`, id, + ) + return scanRun(row) +} + +// ListRuns returns runs, newest first, with optional dag name filter. +func (db *DB) ListRuns(dagName string, limit, offset int) ([]DagRun, int, error) { + var total int + var args []interface{} + where := "" + if dagName != "" { + where = " WHERE dag_name=?" + args = append(args, dagName) + } + err := db.conn.QueryRow("SELECT COUNT(*) FROM dag_runs"+where, args...).Scan(&total) + if err != nil { + return nil, 0, err + } + + query := "SELECT id, dag_name, dag_path, status, trigger, started_at, finished_at, error FROM dag_runs" + + where + " ORDER BY started_at DESC LIMIT ? OFFSET ?" + args = append(args, limit, offset) + + rows, err := db.conn.Query(query, args...) + if err != nil { + return nil, 0, err + } + defer rows.Close() + + var runs []DagRun + for rows.Next() { + r, err := scanRunRows(rows) + if err != nil { + return nil, 0, err + } + runs = append(runs, *r) + } + return runs, total, rows.Err() +} + +// --- DagStepResult CRUD --- + +// DagStepResult mirrors infra.DagStepResult for the store layer. +type DagStepResult struct { + ID string + RunID string + StepName string + Status string + ExitCode int + Stdout string + Stderr string + StartedAt *time.Time + FinishedAt *time.Time + DurationMs int64 + Error string +} + +// InsertStepResult inserts a new step result. +func (db *DB) InsertStepResult(r *DagStepResult) error { + var startedAt, finishedAt *string + if r.StartedAt != nil { + s := r.StartedAt.Format(time.RFC3339) + startedAt = &s + } + if r.FinishedAt != nil { + s := r.FinishedAt.Format(time.RFC3339) + finishedAt = &s + } + _, err := db.conn.Exec( + `INSERT INTO dag_step_results (id, run_id, step_name, status, exit_code, stdout, stderr, started_at, finished_at, duration_ms, error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + r.ID, r.RunID, r.StepName, r.Status, r.ExitCode, r.Stdout, r.Stderr, + startedAt, finishedAt, r.DurationMs, r.Error, + ) + return err +} + +// UpdateStepResult updates a step result by ID. +func (db *DB) UpdateStepResult(id, status string, exitCode int, stdout, stderr string, finishedAt *time.Time, durationMs int64, errMsg string) error { + var fin *string + if finishedAt != nil { + s := finishedAt.Format(time.RFC3339) + fin = &s + } + _, err := db.conn.Exec( + `UPDATE dag_step_results SET status=?, exit_code=?, stdout=?, stderr=?, finished_at=?, duration_ms=?, error=? WHERE id=?`, + status, exitCode, stdout, stderr, fin, durationMs, errMsg, id, + ) + return err +} + +// ListStepResults returns all step results for a given run. +func (db *DB) ListStepResults(runID string) ([]DagStepResult, error) { + rows, err := db.conn.Query( + `SELECT id, run_id, step_name, status, exit_code, stdout, stderr, started_at, finished_at, duration_ms, error + FROM dag_step_results WHERE run_id=? ORDER BY started_at ASC`, runID, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []DagStepResult + for rows.Next() { + var r DagStepResult + var startedAt, finishedAt sql.NullString + if err := rows.Scan(&r.ID, &r.RunID, &r.StepName, &r.Status, &r.ExitCode, + &r.Stdout, &r.Stderr, &startedAt, &finishedAt, &r.DurationMs, &r.Error); err != nil { + return nil, err + } + if startedAt.Valid { + t, _ := time.Parse(time.RFC3339, startedAt.String) + r.StartedAt = &t + } + if finishedAt.Valid { + t, _ := time.Parse(time.RFC3339, finishedAt.String) + r.FinishedAt = &t + } + results = append(results, r) + } + return results, rows.Err() +} + +// --- scan helpers --- + +type scanner interface { + Scan(dest ...interface{}) error +} + +func scanRun(s scanner) (*DagRun, error) { + var r DagRun + var startedAt string + var finishedAt sql.NullString + if err := s.Scan(&r.ID, &r.DagName, &r.DagPath, &r.Status, &r.Trigger, &startedAt, &finishedAt, &r.Error); err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, err + } + r.StartedAt, _ = time.Parse(time.RFC3339, startedAt) + if finishedAt.Valid { + t, _ := time.Parse(time.RFC3339, finishedAt.String) + r.FinishedAt = &t + } + return &r, nil +} + +func scanRunRows(rows *sql.Rows) (*DagRun, error) { + return scanRun(rows) +} diff --git a/dev/feature_flags.json b/dev/feature_flags.json index 376873b6..3b482c2c 100644 --- a/dev/feature_flags.json +++ b/dev/feature_flags.json @@ -1,7 +1,7 @@ { "dag-engine": { - "enabled": false, + "enabled": true, "issue": "0007", - "description": "Sistema propio de orquestacion de DAGs para reemplazar Dagu. Incluye parser YAML, executor con paralelismo, process manager, execution store y scheduler cron." + "description": "Sistema propio de orquestacion de DAGs para reemplazar Dagu. Incluye parser YAML, executor con paralelismo, process manager, execution store SQLite, scheduler cron, CLI y web frontend." } } diff --git a/dev/issues/README.md b/dev/issues/README.md index f889f7c9..13128a14 100644 --- a/dev/issues/README.md +++ b/dev/issues/README.md @@ -8,9 +8,9 @@ | 0004 | Jupyter discover multiple instances | completado | — | feature | — | | 0005 | Jupyter write batch | completado | — | feature | — | | 0006 | Jupyter exec outputs keyerror | completado | — | bugfix | — | -| **0007a** | **DAG engine: core (parse, validate, topo sort)** | pendiente | alta | feature | 0007b-e | -| **0007b** | **DAG engine: process manager (spawn, wait, kill)** | pendiente | alta | feature | 0007e | -| **0007c** | **DAG engine: execution store (SQLite)** | pendiente | alta | feature | 0007e | -| **0007d** | **DAG engine: scheduler (cron parser, ticker)** | pendiente | media | feature | 0007e | -| **0007e** | **DAG engine: app CLI que reemplaza Dagu** | pendiente | alta | feature | — | +| [0007a](completed/0007a-dag-core.md) | DAG engine: core (parse, validate, topo sort) | completado | alta | feature | 0007b-e | +| [0007b](completed/0007b-process-manager.md) | DAG engine: process manager (spawn, wait, kill) | completado | alta | feature | 0007e | +| [0007c](completed/0007c-execution-store.md) | DAG engine: execution store (SQLite) | completado | alta | feature | 0007e | +| [0007d](completed/0007d-scheduler.md) | DAG engine: scheduler (cron match) | completado | media | feature | 0007e | +| [0007e](completed/0007e-dag-executor-app.md) | DAG engine: CLI + web app que reemplaza Dagu | completado | alta | feature | — | | **0008** | **SQLite API Web** | pendiente | alta | feature | — | diff --git a/dev/issues/0007a-dag-core.md b/dev/issues/completed/0007a-dag-core.md similarity index 100% rename from dev/issues/0007a-dag-core.md rename to dev/issues/completed/0007a-dag-core.md diff --git a/dev/issues/0007b-process-manager.md b/dev/issues/completed/0007b-process-manager.md similarity index 100% rename from dev/issues/0007b-process-manager.md rename to dev/issues/completed/0007b-process-manager.md diff --git a/dev/issues/0007c-execution-store.md b/dev/issues/completed/0007c-execution-store.md similarity index 100% rename from dev/issues/0007c-execution-store.md rename to dev/issues/completed/0007c-execution-store.md diff --git a/dev/issues/0007d-scheduler.md b/dev/issues/completed/0007d-scheduler.md similarity index 100% rename from dev/issues/0007d-scheduler.md rename to dev/issues/completed/0007d-scheduler.md diff --git a/dev/issues/0007e-dag-executor-app.md b/dev/issues/completed/0007e-dag-executor-app.md similarity index 100% rename from dev/issues/0007e-dag-executor-app.md rename to dev/issues/completed/0007e-dag-executor-app.md diff --git a/functions/core/cron_match.go b/functions/core/cron_match.go new file mode 100644 index 00000000..01c3d07f --- /dev/null +++ b/functions/core/cron_match.go @@ -0,0 +1,12 @@ +package core + +import "time" + +// CronMatch returns true if time t matches all fields of the cron schedule. +func CronMatch(sched CronSchedule, t time.Time) bool { + return intIn(t.Minute(), sched.Minute) && + intIn(t.Hour(), sched.Hour) && + intIn(t.Day(), sched.DayOfMonth) && + intIn(int(t.Month()), sched.Month) && + intIn(int(t.Weekday()), sched.DayOfWeek) +} diff --git a/functions/core/cron_match.md b/functions/core/cron_match.md new file mode 100644 index 00000000..bceb84b5 --- /dev/null +++ b/functions/core/cron_match.md @@ -0,0 +1,49 @@ +--- +name: cron_match +kind: function +lang: go +domain: core +version: "1.0.0" +purity: pure +signature: "func CronMatch(sched CronSchedule, t time.Time) bool" +description: "Verifica si un instante de tiempo coincide con un cron schedule. Compara los 5 campos (minuto, hora, dia del mes, mes, dia de la semana) y retorna true si todos coinciden." +tags: [cron, scheduling, matching, time, pure] +uses_functions: [] +uses_types: [cron_schedule_go_core] +returns: [] +returns_optional: false +error_type: "" +imports: [time] +params: + - name: sched + desc: "CronSchedule con listas de valores validos por campo (resultado de ParseCronExpr)" + - name: t + desc: "instante de tiempo a verificar contra el schedule" +output: "true si t coincide con todos los campos del cron schedule" +tested: true +tests: + - "9:00 AM coincide con 0 9 * * *" + - "9:15 AM NO coincide con 0 9 * * *" + - "lunes a las 9 coincide con 0 9 * * 1" + - "domingo a las 9 NO coincide con 0 9 * * 1" + - "wildcard * coincide con cualquier valor" + - "specific month" +test_file_path: "functions/core/cron_match_test.go" +file_path: "functions/core/cron_match.go" +--- + +## Ejemplo + +```go +sched, _ := ParseCronExpr("0 9 * * *") +t := time.Date(2026, 4, 11, 9, 0, 0, 0, time.UTC) +CronMatch(sched, t) // true + +t2 := time.Date(2026, 4, 11, 10, 0, 0, 0, time.UTC) +CronMatch(sched, t2) // false +``` + +## Notas + +Funcion pura. Usa AND semantics para day_of_month y day_of_week (ambos deben coincidir), igual que NextCronTime en el mismo paquete. +Reutiliza el helper intIn definido en next_cron_time.go (mismo paquete core). diff --git a/functions/core/cron_match_test.go b/functions/core/cron_match_test.go new file mode 100644 index 00000000..e99e5b2a --- /dev/null +++ b/functions/core/cron_match_test.go @@ -0,0 +1,88 @@ +package core + +import ( + "testing" + "time" +) + +func TestCronMatch(t *testing.T) { + t.Run("9:00 AM coincide con 0 9 * * *", func(t *testing.T) { + sched, err := ParseCronExpr("0 9 * * *") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + ts := time.Date(2026, 4, 11, 9, 0, 0, 0, time.UTC) + if !CronMatch(sched, ts) { + t.Errorf("expected match for 9:00 AM with '0 9 * * *'") + } + }) + + t.Run("9:15 AM NO coincide con 0 9 * * *", func(t *testing.T) { + sched, err := ParseCronExpr("0 9 * * *") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + ts := time.Date(2026, 4, 11, 9, 15, 0, 0, time.UTC) + if CronMatch(sched, ts) { + t.Errorf("expected no match for 9:15 AM with '0 9 * * *'") + } + }) + + t.Run("lunes a las 9 coincide con 0 9 * * 1", func(t *testing.T) { + sched, err := ParseCronExpr("0 9 * * 1") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + // 2026-04-13 is a Monday + ts := time.Date(2026, 4, 13, 9, 0, 0, 0, time.UTC) + if !CronMatch(sched, ts) { + t.Errorf("expected match for Monday 9:00 with '0 9 * * 1'") + } + }) + + t.Run("domingo a las 9 NO coincide con 0 9 * * 1", func(t *testing.T) { + sched, err := ParseCronExpr("0 9 * * 1") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + // 2026-04-12 is a Sunday (weekday 0) + ts := time.Date(2026, 4, 12, 9, 0, 0, 0, time.UTC) + if CronMatch(sched, ts) { + t.Errorf("expected no match for Sunday 9:00 with '0 9 * * 1'") + } + }) + + t.Run("wildcard * coincide con cualquier valor", func(t *testing.T) { + sched, err := ParseCronExpr("*/15 * * * *") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + // 9:30 should match */15 (0,15,30,45) + ts := time.Date(2026, 4, 11, 9, 30, 0, 0, time.UTC) + if !CronMatch(sched, ts) { + t.Errorf("expected match for 9:30 with '*/15 * * * *'") + } + // 9:07 should NOT match */15 + ts2 := time.Date(2026, 4, 11, 9, 7, 0, 0, time.UTC) + if CronMatch(sched, ts2) { + t.Errorf("expected no match for 9:07 with '*/15 * * * *'") + } + }) + + t.Run("specific month", func(t *testing.T) { + sched, err := ParseCronExpr("0 0 1 4 *") + if err != nil { + t.Fatalf("ParseCronExpr: %v", err) + } + // April 1st matches + ts := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + if !CronMatch(sched, ts) { + t.Errorf("expected match for April 1 with '0 0 1 4 *'") + } + // March 1st does NOT match + ts2 := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + if CronMatch(sched, ts2) { + t.Errorf("expected no match for March 1 with '0 0 1 4 *'") + } + }) +} diff --git a/functions/core/dag_definition.go b/functions/core/dag_definition.go new file mode 100644 index 00000000..ee2f2447 --- /dev/null +++ b/functions/core/dag_definition.go @@ -0,0 +1,65 @@ +package core + +// DagContinueOn controls whether a step continues on failure/skip. +type DagContinueOn struct { + Failure bool + Skipped bool +} + +// DagRetryPolicy configures automatic retries for a step. +type DagRetryPolicy struct { + Limit int + IntervalSec int +} + +// DagStep represents a single step in a DAG workflow. +type DagStep struct { + Name string + ID string + Description string + Command string + Script string + Args []string + Shell string + Dir string + Depends []string + Env map[string]string + ContinueOn DagContinueOn + RetryPolicy DagRetryPolicy + TimeoutSec int + Output string + Tags []string +} + +// DagHandlers contains lifecycle handler steps. +type DagHandlers struct { + Init []DagStep + Success []DagStep + Failure []DagStep + Exit []DagStep +} + +// DagDefinition is a complete DAG workflow parsed from YAML. +type DagDefinition struct { + Name string + Description string + Group string + Type string // "graph" or "" (chain/sequential) + WorkingDir string + Shell string + Env map[string]string + Schedule []string + Steps []DagStep + HandlerOn DagHandlers + Tags []string + TimeoutSec int + FilePath string +} + +// DagValidationResult contains validation output. +type DagValidationResult struct { + Valid bool + Errors []string + Warnings []string + Levels [][]string // topological levels (step names/IDs) +} diff --git a/functions/core/dag_parse.go b/functions/core/dag_parse.go new file mode 100644 index 00000000..60c05c5c --- /dev/null +++ b/functions/core/dag_parse.go @@ -0,0 +1,281 @@ +package core + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// rawDagStep is the loosely-typed intermediate representation of a DAG step. +type rawDagStep struct { + Name string `yaml:"name"` + ID string `yaml:"id"` + Description string `yaml:"description"` + Command string `yaml:"command"` + Script string `yaml:"script"` + Args []string `yaml:"args"` + Shell string `yaml:"shell"` + Dir string `yaml:"dir"` + WorkingDir string `yaml:"working_dir"` + Depends []string `yaml:"depends"` + Env interface{} `yaml:"env"` + ContinueOn rawDagContinueOn `yaml:"continue_on"` + RetryPolicy rawDagRetryPolicy `yaml:"retry_policy"` + TimeoutSec int `yaml:"timeout_sec"` + Output string `yaml:"output"` + Tags []string `yaml:"tags"` +} + +type rawDagContinueOn struct { + Failure bool `yaml:"failure"` + Skipped bool `yaml:"skipped"` +} + +type rawDagRetryPolicy struct { + Limit int `yaml:"limit"` + IntervalSec int `yaml:"interval_sec"` +} + +// rawDagHandlers handles both handler_on and handlers aliases. +type rawDagHandlers struct { + Init interface{} `yaml:"init"` + Success interface{} `yaml:"success"` + Failure interface{} `yaml:"failure"` + Exit interface{} `yaml:"exit"` +} + +// rawDag is the loosely-typed intermediate representation of a DAG YAML. +type rawDag struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + Group string `yaml:"group"` + Type string `yaml:"type"` + WorkingDir string `yaml:"working_dir"` + Shell string `yaml:"shell"` + Env interface{} `yaml:"env"` + Schedule interface{} `yaml:"schedule"` + Steps []rawDagStep `yaml:"steps"` + HandlerOn *rawDagHandlers `yaml:"handler_on"` + Handlers *rawDagHandlers `yaml:"handlers"` + Tags []string `yaml:"tags"` + TimeoutSec int `yaml:"timeout_sec"` +} + +// DagParse parses a YAML DAG definition compatible with Dagu format. +// Handles schedule as string or list, env as list of single-key maps, +// handler_on and handlers as aliases, and steps with command/script/depends. +func DagParse(data []byte) (DagDefinition, error) { + var raw rawDag + if err := yaml.Unmarshal(data, &raw); err != nil { + return DagDefinition{}, fmt.Errorf("dag_parse: yaml unmarshal: %w", err) + } + + def := DagDefinition{ + Name: raw.Name, + Description: raw.Description, + Group: raw.Group, + Type: raw.Type, + WorkingDir: raw.WorkingDir, + Shell: raw.Shell, + Tags: raw.Tags, + TimeoutSec: raw.TimeoutSec, + } + + // Normalize env (list of single-key maps or plain map). + dagEnv, err := normalizeEnv(raw.Env) + if err != nil { + return DagDefinition{}, fmt.Errorf("dag_parse: env: %w", err) + } + def.Env = dagEnv + + // Normalize schedule (string or list). + def.Schedule = normalizeSchedule(raw.Schedule) + + // Normalize steps. + steps := make([]DagStep, 0, len(raw.Steps)) + for i, rs := range raw.Steps { + step, err := normalizeStep(rs) + if err != nil { + return DagDefinition{}, fmt.Errorf("dag_parse: step[%d]: %w", i, err) + } + steps = append(steps, step) + } + def.Steps = steps + + // Normalize handlers: handler_on takes precedence, fallback to handlers. + rawHandlers := raw.HandlerOn + if rawHandlers == nil { + rawHandlers = raw.Handlers + } + if rawHandlers != nil { + handlers, err := normalizeHandlers(rawHandlers) + if err != nil { + return DagDefinition{}, fmt.Errorf("dag_parse: handlers: %w", err) + } + def.HandlerOn = handlers + } + + return def, nil +} + +// normalizeSchedule converts schedule from string or []interface{} to []string. +func normalizeSchedule(v interface{}) []string { + if v == nil { + return nil + } + switch t := v.(type) { + case string: + s := strings.TrimSpace(t) + if s == "" { + return nil + } + return []string{s} + case []interface{}: + result := make([]string, 0, len(t)) + for _, item := range t { + if s, ok := item.(string); ok { + s = strings.TrimSpace(s) + if s != "" { + result = append(result, s) + } + } + } + return result + } + return nil +} + +// normalizeEnv converts env from Dagu format (list of single-key maps) or plain map to map[string]string. +func normalizeEnv(v interface{}) (map[string]string, error) { + if v == nil { + return nil, nil + } + switch t := v.(type) { + case map[string]interface{}: + // Plain YAML map. + result := make(map[string]string, len(t)) + for k, val := range t { + result[k] = fmt.Sprintf("%v", val) + } + return result, nil + case []interface{}: + // Dagu format: list of single-key maps [KEY: value, KEY2: value2]. + result := make(map[string]string) + for _, item := range t { + m, ok := item.(map[string]interface{}) + if !ok { + continue + } + for k, val := range m { + result[k] = fmt.Sprintf("%v", val) + } + } + return result, nil + } + return nil, fmt.Errorf("unexpected env type %T", v) +} + +// normalizeStep converts a rawDagStep to a DagStep. +func normalizeStep(rs rawDagStep) (DagStep, error) { + stepEnv, err := normalizeEnv(rs.Env) + if err != nil { + return DagStep{}, fmt.Errorf("env: %w", err) + } + + // working_dir is an alias for dir at the step level. + dir := rs.Dir + if dir == "" { + dir = rs.WorkingDir + } + + return DagStep{ + Name: rs.Name, + ID: rs.ID, + Description: rs.Description, + Command: rs.Command, + Script: rs.Script, + Args: rs.Args, + Shell: rs.Shell, + Dir: dir, + Depends: rs.Depends, + Env: stepEnv, + ContinueOn: DagContinueOn{ + Failure: rs.ContinueOn.Failure, + Skipped: rs.ContinueOn.Skipped, + }, + RetryPolicy: DagRetryPolicy{ + Limit: rs.RetryPolicy.Limit, + IntervalSec: rs.RetryPolicy.IntervalSec, + }, + TimeoutSec: rs.TimeoutSec, + Output: rs.Output, + Tags: rs.Tags, + }, nil +} + +// normalizeHandlers converts rawDagHandlers to DagHandlers. +// Each handler field can be a single step object or a list of steps. +func normalizeHandlers(rh *rawDagHandlers) (DagHandlers, error) { + var h DagHandlers + var err error + h.Init, err = normalizeHandlerField(rh.Init) + if err != nil { + return DagHandlers{}, fmt.Errorf("init: %w", err) + } + h.Success, err = normalizeHandlerField(rh.Success) + if err != nil { + return DagHandlers{}, fmt.Errorf("success: %w", err) + } + h.Failure, err = normalizeHandlerField(rh.Failure) + if err != nil { + return DagHandlers{}, fmt.Errorf("failure: %w", err) + } + h.Exit, err = normalizeHandlerField(rh.Exit) + if err != nil { + return DagHandlers{}, fmt.Errorf("exit: %w", err) + } + return h, nil +} + +// normalizeHandlerField converts a handler field (single step or list) to []DagStep. +func normalizeHandlerField(v interface{}) ([]DagStep, error) { + if v == nil { + return nil, nil + } + + // Re-marshal and unmarshal to handle polymorphic types cleanly. + b, err := yaml.Marshal(v) + if err != nil { + return nil, fmt.Errorf("re-marshal handler: %w", err) + } + + // Try as a list first. + var rawList []rawDagStep + if err := yaml.Unmarshal(b, &rawList); err == nil && len(rawList) > 0 { + steps := make([]DagStep, 0, len(rawList)) + for i, rs := range rawList { + step, err := normalizeStep(rs) + if err != nil { + return nil, fmt.Errorf("step[%d]: %w", i, err) + } + steps = append(steps, step) + } + return steps, nil + } + + // Try as single step. + var rs rawDagStep + if err := yaml.Unmarshal(b, &rs); err != nil { + return nil, fmt.Errorf("unmarshal handler step: %w", err) + } + step, err := normalizeStep(rs) + if err != nil { + return nil, err + } + // If the step has no meaningful content, return nil. + if step.Name == "" && step.ID == "" && step.Command == "" && step.Script == "" { + return nil, nil + } + return []DagStep{step}, nil +} diff --git a/functions/core/dag_parse.md b/functions/core/dag_parse.md new file mode 100644 index 00000000..8b61aae5 --- /dev/null +++ b/functions/core/dag_parse.md @@ -0,0 +1,54 @@ +--- +name: dag_parse +kind: function +lang: go +domain: core +version: "1.0.0" +purity: pure +signature: "func DagParse(data []byte) (DagDefinition, error)" +description: "Parsea YAML de definicion de DAG en formato compatible con Dagu. Soporta schedule como string o lista, env como lista de maps single-key (formato Dagu), handler_on y handlers como aliases, steps con command/script/depends/continue_on, y type graph." +tags: [dag, yaml, parsing, workflow, dagu, pure] +uses_functions: [] +uses_types: [dag_definition_go_core, dag_step_go_core, dag_handlers_go_core] +returns: [] +returns_optional: false +error_type: "" +imports: [fmt, strings, gopkg.in/yaml.v3] +params: + - name: data + desc: "contenido YAML de un archivo de definicion de DAG en formato Dagu" +output: "DagDefinition con todos los campos normalizados; error si el YAML es sintaticamente invalido" +tested: true +tests: + - "parsea DAG simple con steps y depends" + - "parsea schedule como string y como lista" + - "parsea env en formato lista de maps" + - "parsea handler_on y handlers como alias" + - "parsea continue_on y working_dir a nivel step" + - "parsea type graph" +test_file_path: "functions/core/dag_parse_test.go" +file_path: "functions/core/dag_parse.go" +--- + +## Ejemplo + +```go +data := []byte(` +name: mi-dag +schedule: "0 9 * * *" +steps: + - name: hello + command: echo "hello" + - name: world + command: echo "world" + depends: [hello] +`) +dag, err := DagParse(data) +// dag.Name = "mi-dag" +// dag.Schedule = ["0 9 * * *"] +// dag.Steps[1].Depends = ["hello"] +``` + +## Notas + +Funcion pura (el YAML es inmutable, no hay I/O). Internamente usa un struct rawDag para deserializar loosely y luego normaliza campos polimorficos. La estrategia de normalizacion: schedule string->[]string, env lista->map, handlers single-o-lista->[]DagStep. handler_on tiene precedencia sobre handlers si ambos estan presentes. diff --git a/functions/core/dag_parse_test.go b/functions/core/dag_parse_test.go new file mode 100644 index 00000000..bcb4ccc2 --- /dev/null +++ b/functions/core/dag_parse_test.go @@ -0,0 +1,213 @@ +package core + +import ( + "testing" +) + +func TestDagParse(t *testing.T) { + t.Run("parsea DAG simple con steps y depends", func(t *testing.T) { + data := []byte(` +name: example +description: Example workflow +steps: + - name: hello + command: echo "Hello!" + - name: list_files + command: ls -la /tmp + depends: + - hello +`) + dag, err := DagParse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dag.Name != "example" { + t.Errorf("Name: got %q, want %q", dag.Name, "example") + } + if len(dag.Steps) != 2 { + t.Fatalf("Steps: got %d, want 2", len(dag.Steps)) + } + if dag.Steps[0].Name != "hello" { + t.Errorf("Steps[0].Name: got %q, want %q", dag.Steps[0].Name, "hello") + } + if dag.Steps[1].Name != "list_files" { + t.Errorf("Steps[1].Name: got %q, want %q", dag.Steps[1].Name, "list_files") + } + if len(dag.Steps[1].Depends) != 1 || dag.Steps[1].Depends[0] != "hello" { + t.Errorf("Steps[1].Depends: got %v, want [hello]", dag.Steps[1].Depends) + } + }) + + t.Run("parsea schedule como string y como lista", func(t *testing.T) { + // Schedule as string. + dataStr := []byte(` +name: dag-string-schedule +schedule: "0 9 * * 5" +steps: + - name: step1 + command: echo ok +`) + dagStr, err := DagParse(dataStr) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(dagStr.Schedule) != 1 || dagStr.Schedule[0] != "0 9 * * 5" { + t.Errorf("Schedule (string): got %v, want [\"0 9 * * 5\"]", dagStr.Schedule) + } + + // Schedule as list. + dataList := []byte(` +name: dag-list-schedule +schedule: + - "0 9 * * *" + - "0 18 * * *" +steps: + - name: step1 + command: echo ok +`) + dagList, err := DagParse(dataList) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(dagList.Schedule) != 2 { + t.Fatalf("Schedule (list): got %d items, want 2", len(dagList.Schedule)) + } + if dagList.Schedule[0] != "0 9 * * *" { + t.Errorf("Schedule[0]: got %q, want %q", dagList.Schedule[0], "0 9 * * *") + } + if dagList.Schedule[1] != "0 18 * * *" { + t.Errorf("Schedule[1]: got %q, want %q", dagList.Schedule[1], "0 18 * * *") + } + }) + + t.Run("parsea env en formato lista de maps", func(t *testing.T) { + data := []byte(` +name: dag-env +env: + - PROJECT_DIR: /home/lucas/analysis + - PYTHON: /home/lucas/.venv/bin/python +steps: + - name: step1 + command: echo ${PROJECT_DIR} +`) + dag, err := DagParse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dag.Env["PROJECT_DIR"] != "/home/lucas/analysis" { + t.Errorf("Env[PROJECT_DIR]: got %q, want %q", dag.Env["PROJECT_DIR"], "/home/lucas/analysis") + } + if dag.Env["PYTHON"] != "/home/lucas/.venv/bin/python" { + t.Errorf("Env[PYTHON]: got %q, want %q", dag.Env["PYTHON"], "/home/lucas/.venv/bin/python") + } + }) + + t.Run("parsea handler_on y handlers como alias", func(t *testing.T) { + // handler_on with single step object. + dataHandlerOn := []byte(` +name: dag-handler-on +handler_on: + failure: + command: echo "FALLO" +steps: + - name: step1 + command: echo hello +`) + dagHO, err := DagParse(dataHandlerOn) + if err != nil { + t.Fatalf("unexpected error (handler_on): %v", err) + } + if len(dagHO.HandlerOn.Failure) != 1 { + t.Fatalf("HandlerOn.Failure: got %d steps, want 1", len(dagHO.HandlerOn.Failure)) + } + if dagHO.HandlerOn.Failure[0].Command != `echo "FALLO"` { + t.Errorf("HandlerOn.Failure[0].Command: got %q", dagHO.HandlerOn.Failure[0].Command) + } + + // handlers alias with list of steps. + dataHandlers := []byte(` +name: dag-handlers +handlers: + failure: + - name: mark_as_failed + command: echo "FAILED" + success: + - name: notify + command: echo "OK" +steps: + - name: step1 + command: echo hello +`) + dagH, err := DagParse(dataHandlers) + if err != nil { + t.Fatalf("unexpected error (handlers): %v", err) + } + if len(dagH.HandlerOn.Failure) != 1 || dagH.HandlerOn.Failure[0].Name != "mark_as_failed" { + t.Errorf("HandlerOn.Failure: got %v", dagH.HandlerOn.Failure) + } + if len(dagH.HandlerOn.Success) != 1 || dagH.HandlerOn.Success[0].Name != "notify" { + t.Errorf("HandlerOn.Success: got %v", dagH.HandlerOn.Success) + } + }) + + t.Run("parsea continue_on y working_dir a nivel step", func(t *testing.T) { + data := []byte(` +name: dag-step-options +steps: + - id: ingest + description: Procesar archivos + working_dir: /home/lucas/project + command: ./bin/ingest + continue_on: + failure: true + - id: informe + command: python /tmp/script.py + depends: [ingest] +`) + dag, err := DagParse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(dag.Steps) != 2 { + t.Fatalf("Steps: got %d, want 2", len(dag.Steps)) + } + step0 := dag.Steps[0] + if step0.ID != "ingest" { + t.Errorf("Steps[0].ID: got %q, want %q", step0.ID, "ingest") + } + if step0.Dir != "/home/lucas/project" { + t.Errorf("Steps[0].Dir: got %q, want %q", step0.Dir, "/home/lucas/project") + } + if !step0.ContinueOn.Failure { + t.Errorf("Steps[0].ContinueOn.Failure: got false, want true") + } + step1 := dag.Steps[1] + if len(step1.Depends) != 1 || step1.Depends[0] != "ingest" { + t.Errorf("Steps[1].Depends: got %v, want [ingest]", step1.Depends) + } + }) + + t.Run("parsea type graph", func(t *testing.T) { + data := []byte(` +name: dag-graph +type: graph +tags: [finanzas, semanal] +steps: + - name: step1 + command: echo a + - name: step2 + command: echo b + depends: [step1] +`) + dag, err := DagParse(data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dag.Type != "graph" { + t.Errorf("Type: got %q, want %q", dag.Type, "graph") + } + if len(dag.Tags) != 2 { + t.Errorf("Tags: got %v, want [finanzas semanal]", dag.Tags) + } + }) +} diff --git a/functions/core/dag_resolve_env.go b/functions/core/dag_resolve_env.go new file mode 100644 index 00000000..a7a5ae28 --- /dev/null +++ b/functions/core/dag_resolve_env.go @@ -0,0 +1,157 @@ +package core + +import ( + "strings" +) + +// DagResolveEnv resolves environment variable references in a DagDefinition. +// It parses environ (KEY=VALUE pairs from os.Environ()), merges with dag.Env +// (dag.Env takes precedence), and substitutes ${VAR} and $VAR in Command, +// Script, Dir, Args, and Env values of each step. Also substitutes in +// DagDefinition.WorkingDir. Returns a new DagDefinition without mutating the original. +func DagResolveEnv(dag DagDefinition, environ []string) DagDefinition { + // Parse environ into a base map. + base := parseEnviron(environ) + + // Merge dag.Env over base (dag.Env has precedence). + dagEnv := mergeMaps(base, dag.Env) + + // Build new DagDefinition (shallow copy, then replace fields). + result := dag + + // Substitute in WorkingDir. + result.WorkingDir = substitute(dag.WorkingDir, dagEnv) + + // Substitute in Env values of the DAG itself. + result.Env = substituteMap(dag.Env, dagEnv) + + // Resolve steps. + resolvedSteps := make([]DagStep, len(dag.Steps)) + for i, step := range dag.Steps { + // Merge dag env with step env (step env has precedence). + stepEnv := mergeMaps(dagEnv, step.Env) + + rs := step + rs.Command = substitute(step.Command, stepEnv) + rs.Script = substitute(step.Script, stepEnv) + rs.Dir = substitute(step.Dir, stepEnv) + + if len(step.Args) > 0 { + args := make([]string, len(step.Args)) + for j, arg := range step.Args { + args[j] = substitute(arg, stepEnv) + } + rs.Args = args + } + + rs.Env = substituteMap(step.Env, stepEnv) + resolvedSteps[i] = rs + } + result.Steps = resolvedSteps + + return result +} + +// parseEnviron parses a slice of "KEY=VALUE" strings into a map. +func parseEnviron(environ []string) map[string]string { + m := make(map[string]string, len(environ)) + for _, kv := range environ { + idx := strings.IndexByte(kv, '=') + if idx < 0 { + continue + } + m[kv[:idx]] = kv[idx+1:] + } + return m +} + +// mergeMaps returns a new map with all entries from base, then overlaid with overlay. +func mergeMaps(base, overlay map[string]string) map[string]string { + result := make(map[string]string, len(base)+len(overlay)) + for k, v := range base { + result[k] = v + } + for k, v := range overlay { + result[k] = v + } + return result +} + +// substituteMap applies substitute to all values in a map. +func substituteMap(m map[string]string, env map[string]string) map[string]string { + if len(m) == 0 { + return m + } + result := make(map[string]string, len(m)) + for k, v := range m { + result[k] = substitute(v, env) + } + return result +} + +// substitute replaces ${VAR} and $VAR occurrences in s using the env map. +func substitute(s string, env map[string]string) string { + if s == "" || !strings.Contains(s, "$") { + return s + } + + var b strings.Builder + i := 0 + for i < len(s) { + if s[i] != '$' { + b.WriteByte(s[i]) + i++ + continue + } + + // Found '$' + i++ + if i >= len(s) { + b.WriteByte('$') + break + } + + if s[i] == '{' { + // ${VAR} form. + i++ + end := strings.IndexByte(s[i:], '}') + if end < 0 { + // No closing brace — write as-is. + b.WriteString("${") + continue + } + varName := s[i : i+end] + i += end + 1 + if val, ok := env[varName]; ok { + b.WriteString(val) + } else { + b.WriteString("${") + b.WriteString(varName) + b.WriteByte('}') + } + } else if isEnvVarChar(s[i]) { + // $VAR form (bare variable name). + start := i + for i < len(s) && isEnvVarChar(s[i]) { + i++ + } + varName := s[start:i] + if val, ok := env[varName]; ok { + b.WriteString(val) + } else { + b.WriteByte('$') + b.WriteString(varName) + } + } else { + // Not a valid variable start — keep '$' and continue. + b.WriteByte('$') + } + } + + return b.String() +} + +// isEnvVarChar returns true for characters valid in environment variable names. +func isEnvVarChar(c byte) bool { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_' +} diff --git a/functions/core/dag_resolve_env.md b/functions/core/dag_resolve_env.md new file mode 100644 index 00000000..dbe9180e --- /dev/null +++ b/functions/core/dag_resolve_env.md @@ -0,0 +1,39 @@ +--- +name: dag_resolve_env +kind: function +lang: go +domain: core +version: "1.0.0" +purity: pure +signature: "func DagResolveEnv(dag DagDefinition, environ []string) DagDefinition" +description: "Resuelve referencias a variables de entorno (${VAR} y $VAR) en un DagDefinition. Mergea environ del sistema con dag.Env (dag.Env tiene precedencia), y sustituye en Command, Script, Dir, Args y Env values de cada step. No muta el DagDefinition original." +tags: [dag, env, substitution, variable-expansion, pure] +uses_functions: [] +uses_types: [dag_definition_go_core, dag_step_go_core] +returns: [] +returns_optional: false +error_type: "" +imports: [strings] +params: + - name: dag + desc: "DagDefinition parseado con posibles referencias ${VAR} en sus campos" + - name: environ + desc: "lista de strings KEY=VALUE del entorno del sistema (tipicamente os.Environ())" +output: "nuevo DagDefinition con todas las referencias de variables sustituidas por sus valores" +tested: false +tests: [] +test_file_path: "" +file_path: "functions/core/dag_resolve_env.go" +--- + +## Ejemplo + +```go +dag, _ := DagParse(yamlData) +resolved := DagResolveEnv(dag, os.Environ()) +// resolved.Steps[0].Command tiene ${PROJECT_DIR} sustituido +``` + +## Notas + +Funcion pura. No muta el DagDefinition de entrada — retorna una copia con los campos resueltos. Precedencia de variables: environ base < dag.Env < step.Env. Sustituye tanto ${VAR} (con llaves) como $VAR (bare). Si una variable no existe en el entorno, se deja sin sustituir. Los campos sustituidos son: DagDefinition.WorkingDir, Env values del DAG, y por step: Command, Script, Dir, Args, Env values. diff --git a/functions/core/dag_topo_sort.go b/functions/core/dag_topo_sort.go new file mode 100644 index 00000000..936752a3 --- /dev/null +++ b/functions/core/dag_topo_sort.go @@ -0,0 +1,78 @@ +package core + +import "fmt" + +// DagTopoSort performs a topological sort of DAG steps using Kahn's algorithm. +// Returns levels where each level contains steps that can run in parallel. +// Returns an error if a dependency cycle is detected. +func DagTopoSort(steps []DagStep) ([][]DagStep, error) { + if len(steps) == 0 { + return nil, nil + } + + // Build index: ref -> DagStep. + index := make(map[string]DagStep, len(steps)) + for _, s := range steps { + index[stepRef(s)] = s + } + + // Compute in-degree for each step. + inDegree := make(map[string]int, len(steps)) + for _, s := range steps { + ref := stepRef(s) + if _, ok := inDegree[ref]; !ok { + inDegree[ref] = 0 + } + for range s.Depends { + inDegree[ref]++ // ref depends on a dep, so ref gets +1 + } + } + + // Build adjacency list: dep -> list of steps that depend on dep. + adj := make(map[string][]string, len(steps)) + for _, s := range steps { + ref := stepRef(s) + for _, dep := range s.Depends { + adj[dep] = append(adj[dep], ref) + } + } + + // Initialize queue with steps that have in-degree 0. + var queue []string + for _, s := range steps { + ref := stepRef(s) + if inDegree[ref] == 0 { + queue = append(queue, ref) + } + } + + var levels [][]DagStep + processed := 0 + + for len(queue) > 0 { + // The current queue forms a parallel level. + level := make([]DagStep, 0, len(queue)) + nextQueue := []string{} + + for _, ref := range queue { + level = append(level, index[ref]) + processed++ + // Reduce in-degree for dependents. + for _, dependent := range adj[ref] { + inDegree[dependent]-- + if inDegree[dependent] == 0 { + nextQueue = append(nextQueue, dependent) + } + } + } + + levels = append(levels, level) + queue = nextQueue + } + + if processed != len(steps) { + return nil, fmt.Errorf("dag_topo_sort: cycle detected (%d of %d steps processed)", processed, len(steps)) + } + + return levels, nil +} diff --git a/functions/core/dag_topo_sort.md b/functions/core/dag_topo_sort.md new file mode 100644 index 00000000..645976ae --- /dev/null +++ b/functions/core/dag_topo_sort.md @@ -0,0 +1,48 @@ +--- +name: dag_topo_sort +kind: function +lang: go +domain: core +version: "1.0.0" +purity: pure +signature: "func DagTopoSort(steps []DagStep) ([][]DagStep, error)" +description: "Ordenamiento topologico de steps de un DAG usando el algoritmo de Kahn. Retorna niveles donde cada nivel contiene steps que pueden ejecutarse en paralelo. Detecta ciclos y retorna error si los hay." +tags: [dag, topological-sort, graph, kahn, pure] +uses_functions: [] +uses_types: [dag_step_go_core] +returns: [] +returns_optional: false +error_type: "" +imports: [fmt] +params: + - name: steps + desc: "lista de DagStep con sus dependencias definidas en el campo Depends" +output: "slice de niveles donde cada nivel es un slice de DagStep que pueden ejecutarse en paralelo; error si hay ciclo" +tested: true +tests: + - "cadena lineal A->B->C produce tres niveles" + - "diamond A->B A->C B->D C->D produce cuatro niveles" + - "steps paralelos sin depends produce un solo nivel" + - "ciclo A->B->A retorna error" +test_file_path: "functions/core/dag_topo_sort_test.go" +file_path: "functions/core/dag_topo_sort.go" +--- + +## Ejemplo + +```go +steps := []DagStep{ + {Name: "a"}, + {Name: "b", Depends: []string{"a"}}, + {Name: "c", Depends: []string{"a"}}, + {Name: "d", Depends: []string{"b", "c"}}, +} +levels, err := DagTopoSort(steps) +// levels[0] = [a] +// levels[1] = [b, c] (paralelo) +// levels[2] = [d] +``` + +## Notas + +Funcion pura. Implementa Kahn's algorithm: calcula in-degree inicial, pone en cola los steps con in-degree 0, extrae un nivel completo por iteracion y reduce el in-degree de los dependientes. Si al terminar hay steps sin procesar, hay un ciclo. El orden dentro de cada nivel no esta garantizado — depende del orden del slice de entrada. diff --git a/functions/core/dag_topo_sort_test.go b/functions/core/dag_topo_sort_test.go new file mode 100644 index 00000000..ce269e9b --- /dev/null +++ b/functions/core/dag_topo_sort_test.go @@ -0,0 +1,95 @@ +package core + +import ( + "testing" +) + +func TestDagTopoSort(t *testing.T) { + t.Run("cadena lineal A->B->C produce tres niveles", func(t *testing.T) { + steps := []DagStep{ + {Name: "a"}, + {Name: "b", Depends: []string{"a"}}, + {Name: "c", Depends: []string{"b"}}, + } + levels, err := DagTopoSort(steps) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(levels) != 3 { + t.Fatalf("levels: got %d, want 3", len(levels)) + } + if len(levels[0]) != 1 || levels[0][0].Name != "a" { + t.Errorf("levels[0]: got %v, want [a]", stepNames(levels[0])) + } + if len(levels[1]) != 1 || levels[1][0].Name != "b" { + t.Errorf("levels[1]: got %v, want [b]", stepNames(levels[1])) + } + if len(levels[2]) != 1 || levels[2][0].Name != "c" { + t.Errorf("levels[2]: got %v, want [c]", stepNames(levels[2])) + } + }) + + t.Run("diamond A->B A->C B->D C->D produce cuatro niveles", func(t *testing.T) { + steps := []DagStep{ + {Name: "a"}, + {Name: "b", Depends: []string{"a"}}, + {Name: "c", Depends: []string{"a"}}, + {Name: "d", Depends: []string{"b", "c"}}, + } + levels, err := DagTopoSort(steps) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Expected: level0=[a], level1=[b,c], level2=[d] + if len(levels) != 3 { + t.Fatalf("levels: got %d, want 3", len(levels)) + } + if len(levels[0]) != 1 || levels[0][0].Name != "a" { + t.Errorf("levels[0]: got %v, want [a]", stepNames(levels[0])) + } + if len(levels[1]) != 2 { + t.Errorf("levels[1]: got %v, want 2 steps", stepNames(levels[1])) + } + if len(levels[2]) != 1 || levels[2][0].Name != "d" { + t.Errorf("levels[2]: got %v, want [d]", stepNames(levels[2])) + } + }) + + t.Run("steps paralelos sin depends produce un solo nivel", func(t *testing.T) { + steps := []DagStep{ + {Name: "x"}, + {Name: "y"}, + {Name: "z"}, + } + levels, err := DagTopoSort(steps) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(levels) != 1 { + t.Fatalf("levels: got %d, want 1", len(levels)) + } + if len(levels[0]) != 3 { + t.Errorf("levels[0]: got %d steps, want 3", len(levels[0])) + } + }) + + t.Run("ciclo A->B->A retorna error", func(t *testing.T) { + steps := []DagStep{ + {Name: "a", Depends: []string{"b"}}, + {Name: "b", Depends: []string{"a"}}, + } + _, err := DagTopoSort(steps) + if err == nil { + t.Fatal("expected error for cycle, got nil") + } + }) +} + +// stepNames extracts step names for readable test output. +func stepNames(steps []DagStep) []string { + names := make([]string, len(steps)) + for i, s := range steps { + names[i] = stepRef(s) + } + return names +} diff --git a/functions/core/dag_validate.go b/functions/core/dag_validate.go new file mode 100644 index 00000000..2c8c077a --- /dev/null +++ b/functions/core/dag_validate.go @@ -0,0 +1,83 @@ +package core + +import "fmt" + +// DagValidate validates a DagDefinition for structural correctness. +// Checks: steps have name/ID, no duplicate names/IDs, all depends reference +// existing steps, no dependency cycles. On success, computes topological levels. +// Returns warnings for steps with both command and script set. +func DagValidate(dag DagDefinition) DagValidationResult { + result := DagValidationResult{Valid: true} + + // Build name/ID sets and check for missing identifiers and duplicates. + seen := make(map[string]bool) + for i, step := range dag.Steps { + ref := stepRef(step) + if ref == "" { + result.Errors = append(result.Errors, + fmt.Sprintf("step[%d]: must have name or id", i)) + result.Valid = false + continue + } + if seen[ref] { + result.Errors = append(result.Errors, + fmt.Sprintf("step[%d]: duplicate name/id %q", i, ref)) + result.Valid = false + } + seen[ref] = true + + // Warning: command and script both set. + if step.Command != "" && step.Script != "" { + result.Warnings = append(result.Warnings, + fmt.Sprintf("step %q: has both command and script", ref)) + } + } + + if !result.Valid { + return result + } + + // Check that all depends reference existing steps. + for _, step := range dag.Steps { + for _, dep := range step.Depends { + if !seen[dep] { + result.Errors = append(result.Errors, + fmt.Sprintf("step %q: depends on unknown step %q", stepRef(step), dep)) + result.Valid = false + } + } + } + + if !result.Valid { + return result + } + + // Topological sort with Kahn's — detects cycles and computes levels. + levels, err := DagTopoSort(dag.Steps) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("cycle detected: %v", err)) + result.Valid = false + return result + } + + // Convert [][]DagStep to [][]string for the result. + strLevels := make([][]string, len(levels)) + for i, level := range levels { + refs := make([]string, len(level)) + for j, s := range level { + refs[j] = stepRef(s) + } + strLevels[i] = refs + } + result.Levels = strLevels + + return result +} + +// stepRef returns the canonical reference for a step (ID preferred, then Name). +func stepRef(s DagStep) string { + if s.ID != "" { + return s.ID + } + return s.Name +} diff --git a/functions/core/dag_validate.md b/functions/core/dag_validate.md new file mode 100644 index 00000000..6ce46dc1 --- /dev/null +++ b/functions/core/dag_validate.md @@ -0,0 +1,46 @@ +--- +name: dag_validate +kind: function +lang: go +domain: core +version: "1.0.0" +purity: pure +signature: "func DagValidate(dag DagDefinition) DagValidationResult" +description: "Valida un DagDefinition para correcto uso estructural. Verifica que cada step tenga nombre o ID, que no haya duplicados, que todos los depends referencien steps existentes y que no haya ciclos (algoritmo de Kahn). Si el DAG es valido, calcula los niveles topologicos." +tags: [dag, validation, workflow, graph, pure] +uses_functions: [dag_topo_sort_go_core] +uses_types: [dag_definition_go_core, dag_validation_result_go_core] +returns: [] +returns_optional: false +error_type: "" +imports: [fmt] +params: + - name: dag + desc: "DagDefinition parseado y a validar" +output: "DagValidationResult con Valid=true si no hay errores, Errors con los problemas encontrados, Warnings con avisos no fatales, y Levels con los niveles topologicos si el DAG es valido" +tested: true +tests: + - "DAG valido retorna Valid true y levels calculados" + - "step sin nombre retorna error" + - "depends a step inexistente retorna error" + - "ciclo en dependencias retorna error" +test_file_path: "functions/core/dag_validate_test.go" +file_path: "functions/core/dag_validate.go" +--- + +## Ejemplo + +```go +dag, _ := DagParse(yamlData) +res := DagValidate(dag) +if !res.Valid { + for _, e := range res.Errors { + fmt.Println("ERROR:", e) + } +} +// res.Levels = [["step-a"], ["step-b", "step-c"], ["step-d"]] +``` + +## Notas + +Funcion pura. No modifica el DagDefinition de entrada. El calculo de niveles topologicos se delega a DagTopoSort para mantener la separacion de responsabilidades. Un warning (command+script simultaneos) no invalida el DAG. diff --git a/functions/core/dag_validate_test.go b/functions/core/dag_validate_test.go new file mode 100644 index 00000000..28b8f13b --- /dev/null +++ b/functions/core/dag_validate_test.go @@ -0,0 +1,113 @@ +package core + +import ( + "testing" +) + +func TestDagValidate(t *testing.T) { + t.Run("DAG valido retorna Valid true y levels calculados", func(t *testing.T) { + dag := DagDefinition{ + Name: "valid-dag", + Steps: []DagStep{ + {Name: "a", Command: "echo a"}, + {Name: "b", Command: "echo b", Depends: []string{"a"}}, + {Name: "c", Command: "echo c", Depends: []string{"a"}}, + {Name: "d", Command: "echo d", Depends: []string{"b", "c"}}, + }, + } + res := DagValidate(dag) + if !res.Valid { + t.Errorf("Valid: got false, want true. Errors: %v", res.Errors) + } + if len(res.Errors) != 0 { + t.Errorf("Errors: got %v, want empty", res.Errors) + } + // Should have 3 levels: [a], [b,c], [d] + if len(res.Levels) != 3 { + t.Errorf("Levels: got %d, want 3. Levels: %v", len(res.Levels), res.Levels) + } + if len(res.Levels[0]) != 1 || res.Levels[0][0] != "a" { + t.Errorf("Levels[0]: got %v, want [a]", res.Levels[0]) + } + if len(res.Levels[2]) != 1 || res.Levels[2][0] != "d" { + t.Errorf("Levels[2]: got %v, want [d]", res.Levels[2]) + } + }) + + t.Run("step sin nombre retorna error", func(t *testing.T) { + dag := DagDefinition{ + Name: "bad-dag", + Steps: []DagStep{ + {Command: "echo hello"}, // no name, no ID + }, + } + res := DagValidate(dag) + if res.Valid { + t.Error("Valid: got true, want false") + } + if len(res.Errors) == 0 { + t.Error("Errors: got empty, want at least one error") + } + }) + + t.Run("depends a step inexistente retorna error", func(t *testing.T) { + dag := DagDefinition{ + Name: "bad-deps-dag", + Steps: []DagStep{ + {Name: "step1", Command: "echo ok"}, + {Name: "step2", Command: "echo ok", Depends: []string{"nonexistent"}}, + }, + } + res := DagValidate(dag) + if res.Valid { + t.Error("Valid: got true, want false") + } + found := false + for _, e := range res.Errors { + if containsStr(e, "nonexistent") { + found = true + break + } + } + if !found { + t.Errorf("Errors should mention 'nonexistent', got: %v", res.Errors) + } + }) + + t.Run("ciclo en dependencias retorna error", func(t *testing.T) { + dag := DagDefinition{ + Name: "cyclic-dag", + Steps: []DagStep{ + {Name: "a", Command: "echo a", Depends: []string{"b"}}, + {Name: "b", Command: "echo b", Depends: []string{"a"}}, + }, + } + res := DagValidate(dag) + if res.Valid { + t.Error("Valid: got true, want false") + } + found := false + for _, e := range res.Errors { + if containsStr(e, "cycle") { + found = true + break + } + } + if !found { + t.Errorf("Errors should mention 'cycle', got: %v", res.Errors) + } + }) +} + +// containsStr returns true if s contains substr. +func containsStr(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + func() bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false + }()) +} diff --git a/functions/infra/dag_run.go b/functions/infra/dag_run.go new file mode 100644 index 00000000..6e67705e --- /dev/null +++ b/functions/infra/dag_run.go @@ -0,0 +1,30 @@ +package infra + +import "time" + +// DagRun represents one execution of a DAG workflow. +type DagRun struct { + ID string + DagName string + DagPath string + Status string // pending, running, success, failed, cancelled + StartedAt time.Time + FinishedAt time.Time + Trigger string // manual, cron, api + Error string +} + +// DagStepResult represents the outcome of one step within a DagRun. +type DagStepResult struct { + ID string + RunID string + StepName string + Status string // pending, running, success, failed, skipped + ExitCode int + Stdout string + Stderr string + StartedAt time.Time + FinishedAt time.Time + DurationMs int64 + Error string +} diff --git a/functions/infra/process_handle.go b/functions/infra/process_handle.go new file mode 100644 index 00000000..f7afffcf --- /dev/null +++ b/functions/infra/process_handle.go @@ -0,0 +1,26 @@ +package infra + +import ( + "bytes" + "os/exec" + "time" +) + +// ProcessHandle represents a running subprocess with output buffers. +type ProcessHandle struct { + Cmd *exec.Cmd + Pid int + StartTime time.Time + Dir string + stdout *bytes.Buffer + stderr *bytes.Buffer +} + +// ProcessResult contains the outcome of a completed subprocess. +type ProcessResult struct { + ExitCode int + Stdout string + Stderr string + DurationMs int64 + Killed bool +} diff --git a/functions/infra/process_kill.go b/functions/infra/process_kill.go new file mode 100644 index 00000000..3c47fecc --- /dev/null +++ b/functions/infra/process_kill.go @@ -0,0 +1,42 @@ +package infra + +import ( + "fmt" + "syscall" + "time" +) + +// ProcessKill sends SIGTERM to the process group of handle, then waits up to +// graceSec seconds for the process to exit. If it is still alive after the +// grace period, SIGKILL is sent. Returns an error only if the signal could not +// be delivered (e.g. the process group does not exist). +func ProcessKill(handle *ProcessHandle, graceSec int) error { + // Send SIGTERM to the process group (negative pid targets the group). + if err := syscall.Kill(-handle.Pid, syscall.SIGTERM); err != nil { + // ESRCH means the process is already gone — not an error from our view. + if err != syscall.ESRCH { + return fmt.Errorf("process_kill: sigterm: %w", err) + } + return nil + } + + // Poll until the process exits or the grace period expires. + deadline := time.Now().Add(time.Duration(graceSec) * time.Second) + for time.Now().Before(deadline) { + // Check if process has exited by sending signal 0 (no-op). + err := syscall.Kill(-handle.Pid, 0) + if err == syscall.ESRCH { + // Process group is gone. + return nil + } + time.Sleep(100 * time.Millisecond) + } + + // Still alive after grace period — escalate to SIGKILL. + if err := syscall.Kill(-handle.Pid, syscall.SIGKILL); err != nil { + if err != syscall.ESRCH { + return fmt.Errorf("process_kill: sigkill: %w", err) + } + } + return nil +} diff --git a/functions/infra/process_kill.md b/functions/infra/process_kill.md new file mode 100644 index 00000000..6ce9fb0c --- /dev/null +++ b/functions/infra/process_kill.md @@ -0,0 +1,45 @@ +--- +name: process_kill +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func ProcessKill(handle *ProcessHandle, graceSec int) error" +description: "Termina un subproceso enviando SIGTERM al process group. Espera hasta graceSec segundos a que el proceso muera voluntariamente. Si sigue vivo, envia SIGKILL. Retorna error solo si la senal no pudo entregarse." +tags: [process, subprocess, kill, signal, sigterm, sigkill, infra] +uses_functions: [] +uses_types: [process_handle_go_infra] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [fmt, syscall, time] +params: + - name: handle + desc: "handle del proceso lanzado por ProcessSpawn" + - name: graceSec + desc: "segundos de gracia entre SIGTERM y SIGKILL; 0 envia SIGKILL inmediatamente" +output: "nil si el proceso fue terminado correctamente; error si la senal no pudo entregarse" +tested: true +tests: + - "kill process" +test_file_path: "functions/infra/process_spawn_test.go" +file_path: "functions/infra/process_kill.go" +--- + +## Ejemplo + +```go +h, err := ProcessSpawn("sleep 300", "", nil, "") +if err != nil { + log.Fatal(err) +} +// Dar 3 segundos de gracia antes de SIGKILL +if err := ProcessKill(h, 3); err != nil { + log.Printf("kill failed: %v", err) +} +``` + +## Notas + +Funcion impura: envia senales al sistema operativo. Usa -handle.Pid (negativo) para direccionar el process group completo, matando tanto al proceso principal como a sus hijos. ESRCH se ignora porque significa que el proceso ya murio, lo cual es el objetivo deseado. Comprueba si el proceso sigue vivo con signal 0 (kill -0) cada 100ms durante el grace period. diff --git a/functions/infra/process_spawn.go b/functions/infra/process_spawn.go new file mode 100644 index 00000000..f2a33364 --- /dev/null +++ b/functions/infra/process_spawn.go @@ -0,0 +1,74 @@ +package infra + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +// ProcessSpawn launches a subprocess using the given shell. +// If shell is empty, "sh" is used. If command contains newlines it is treated +// as a multi-line script: the content is written to a temp file and executed +// with `shell `. Otherwise it is executed with `shell -c `. +// dir sets the working directory (empty = inherit). env sets the environment +// (nil = inherit parent env). The process group is created with Setpgid so +// that ProcessKill can target the whole group. +func ProcessSpawn(command string, dir string, env []string, shell string) (*ProcessHandle, error) { + if shell == "" { + shell = "sh" + } + + var cmd *exec.Cmd + + if strings.Contains(command, "\n") { + // Multi-line script: write to a temp file and execute it. + tmp, err := os.CreateTemp("", "fn-proc-*.sh") + if err != nil { + return nil, fmt.Errorf("process_spawn: create temp file: %w", err) + } + if _, err := tmp.WriteString(command); err != nil { + _ = os.Remove(tmp.Name()) + return nil, fmt.Errorf("process_spawn: write temp file: %w", err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmp.Name()) + return nil, fmt.Errorf("process_spawn: close temp file: %w", err) + } + cmd = exec.Command(shell, tmp.Name()) + } else { + cmd = exec.Command(shell, "-c", command) + } + + if dir != "" { + cmd.Dir = dir + } + if len(env) > 0 { + cmd.Env = env + } + + // New process group so we can kill all children as a group. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + // Use buffers instead of pipes to avoid race between Wait() and ReadAll(). + var stdoutBuf, stderrBuf bytes.Buffer + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + + start := time.Now() + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("process_spawn: start: %w", err) + } + + return &ProcessHandle{ + Cmd: cmd, + Pid: cmd.Process.Pid, + StartTime: start, + Dir: dir, + stdout: &stdoutBuf, + stderr: &stderrBuf, + }, nil +} diff --git a/functions/infra/process_spawn.md b/functions/infra/process_spawn.md new file mode 100644 index 00000000..ec7c439b --- /dev/null +++ b/functions/infra/process_spawn.md @@ -0,0 +1,52 @@ +--- +name: process_spawn +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func ProcessSpawn(command string, dir string, env []string, shell string) (*ProcessHandle, error)" +description: "Lanza un subproceso usando el shell indicado. Si shell esta vacio usa 'sh'. Comandos con newlines se tratan como scripts multilinea (se escriben a un archivo temporal). Configura un process group propio (Setpgid) para poder matar todos los hijos con ProcessKill. Captura stdout y stderr via pipes." +tags: [process, subprocess, spawn, exec, shell, infra] +uses_functions: [] +uses_types: [process_handle_go_infra] +returns: [process_handle_go_infra] +returns_optional: false +error_type: "error_go_core" +imports: [fmt, os, os/exec, strings, syscall, time] +params: + - name: command + desc: "comando shell a ejecutar; si contiene newlines se trata como script multilinea" + - name: dir + desc: "directorio de trabajo del proceso hijo; vacio hereda el del proceso padre" + - name: env + desc: "variables de entorno en formato KEY=VALUE; nil hereda el entorno del proceso padre" + - name: shell + desc: "interprete shell a usar (sh, bash, zsh); vacio usa 'sh'" +output: "handle del proceso lanzado con Cmd, Pid, StartTime, Dir y los pipes de I/O" +tested: true +tests: + - "spawn and wait echo" + - "spawn with timeout kills" + - "spawn with env" + - "spawn script" + - "spawn with working dir" + - "kill process" +test_file_path: "functions/infra/process_spawn_test.go" +file_path: "functions/infra/process_spawn.go" +--- + +## Ejemplo + +```go +h, err := ProcessSpawn("echo hello", "", nil, "") +if err != nil { + log.Fatal(err) +} +res, err := ProcessWait(h, 10) +fmt.Println(res.Stdout) // "hello\n" +``` + +## Notas + +Funcion impura: hace I/O (crea archivo temporal para scripts, lanza proceso). El process group (Setpgid=true) permite a ProcessKill enviar senales al grupo completo con -Pid, afectando a todos los hijos del proceso lanzado. Para scripts multilinea el archivo temporal queda en el directorio temporal del OS y no se limpia automaticamente. diff --git a/functions/infra/process_spawn_test.go b/functions/infra/process_spawn_test.go new file mode 100644 index 00000000..d3918044 --- /dev/null +++ b/functions/infra/process_spawn_test.go @@ -0,0 +1,107 @@ +package infra + +import ( + "strings" + "testing" +) + +func TestProcessSpawn(t *testing.T) { + t.Run("spawn and wait echo", func(t *testing.T) { + h, err := ProcessSpawn("echo hello", "", nil, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + res, err := ProcessWait(h, 10) + if err != nil { + t.Fatalf("wait: %v", err) + } + if res.ExitCode != 0 { + t.Errorf("exit code: got %d, want 0", res.ExitCode) + } + if !strings.Contains(res.Stdout, "hello") { + t.Errorf("stdout: got %q, want it to contain 'hello'", res.Stdout) + } + }) + + t.Run("spawn with timeout kills", func(t *testing.T) { + h, err := ProcessSpawn("sleep 60", "", nil, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + res, err := ProcessWait(h, 2) + if err != nil { + t.Fatalf("wait: %v", err) + } + if !res.Killed { + t.Errorf("killed: got false, want true") + } + if res.ExitCode == 0 { + t.Errorf("exit code: got 0, want != 0 after kill") + } + }) + + t.Run("spawn with env", func(t *testing.T) { + h, err := ProcessSpawn("echo $TEST_VAR", "", []string{"PATH=/usr/bin:/bin", "TEST_VAR=hello123"}, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + res, err := ProcessWait(h, 10) + if err != nil { + t.Fatalf("wait: %v", err) + } + if !strings.Contains(res.Stdout, "hello123") { + t.Errorf("stdout: got %q, want it to contain 'hello123'", res.Stdout) + } + }) + + t.Run("spawn script", func(t *testing.T) { + script := "#!/bin/sh\necho line1\necho line2" + h, err := ProcessSpawn(script, "", nil, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + res, err := ProcessWait(h, 10) + if err != nil { + t.Fatalf("wait: %v", err) + } + if !strings.Contains(res.Stdout, "line1") { + t.Errorf("stdout: got %q, want it to contain 'line1'", res.Stdout) + } + if !strings.Contains(res.Stdout, "line2") { + t.Errorf("stdout: got %q, want it to contain 'line2'", res.Stdout) + } + }) + + t.Run("spawn with working dir", func(t *testing.T) { + h, err := ProcessSpawn("pwd", "/tmp", nil, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + res, err := ProcessWait(h, 10) + if err != nil { + t.Fatalf("wait: %v", err) + } + if !strings.Contains(res.Stdout, "/tmp") { + t.Errorf("stdout: got %q, want it to contain '/tmp'", res.Stdout) + } + }) + + t.Run("kill process", func(t *testing.T) { + h, err := ProcessSpawn("sleep 60", "", nil, "") + if err != nil { + t.Fatalf("spawn: %v", err) + } + if err := ProcessKill(h, 1); err != nil { + t.Fatalf("kill: %v", err) + } + // After kill, Wait should unblock quickly. + _ = h.Cmd.Wait() + state := h.Cmd.ProcessState + if state == nil { + t.Fatal("process state is nil after kill+wait") + } + if state.ExitCode() == 0 { + t.Errorf("exit code: got 0 after kill, want non-zero") + } + }) +} diff --git a/functions/infra/process_wait.go b/functions/infra/process_wait.go new file mode 100644 index 00000000..6680533f --- /dev/null +++ b/functions/infra/process_wait.go @@ -0,0 +1,51 @@ +package infra + +import ( + "time" +) + +// ProcessWait waits for a subprocess to finish and collects its output. +// If timeoutSec > 0 and the process has not exited by then, ProcessKill is +// called with graceSec=5 and the result is marked Killed=true. +func ProcessWait(handle *ProcessHandle, timeoutSec int) (ProcessResult, error) { + // Wait for the process in a goroutine. + waitCh := make(chan error, 1) + go func() { + waitCh <- handle.Cmd.Wait() + }() + + killed := false + + if timeoutSec > 0 { + timer := time.NewTimer(time.Duration(timeoutSec) * time.Second) + defer timer.Stop() + select { + case <-timer.C: + // Timeout exceeded — kill the process group. + _ = ProcessKill(handle, 5) + killed = true + <-waitCh + case <-waitCh: + } + } else { + <-waitCh + } + + // After Wait() returns, buffers are safe to read. + exitCode := 0 + if handle.Cmd.ProcessState != nil { + exitCode = handle.Cmd.ProcessState.ExitCode() + } else if killed { + exitCode = -1 + } + + duration := time.Since(handle.StartTime) + + return ProcessResult{ + ExitCode: exitCode, + Stdout: handle.stdout.String(), + Stderr: handle.stderr.String(), + DurationMs: duration.Milliseconds(), + Killed: killed, + }, nil +} diff --git a/functions/infra/process_wait.md b/functions/infra/process_wait.md new file mode 100644 index 00000000..5153a4ee --- /dev/null +++ b/functions/infra/process_wait.md @@ -0,0 +1,49 @@ +--- +name: process_wait +kind: function +lang: go +domain: infra +version: "1.0.0" +purity: impure +signature: "func ProcessWait(handle *ProcessHandle, timeoutSec int) (ProcessResult, error)" +description: "Espera a que un subproceso termine y recopila su salida. Lee stdout y stderr completos en goroutines para evitar deadlocks en pipes. Si timeoutSec > 0 y el proceso no termina en ese tiempo, llama a ProcessKill y marca el resultado con Killed=true. Retorna el exit code, salida completa y duracion total." +tags: [process, subprocess, wait, timeout, exec, infra] +uses_functions: [process_kill_go_infra] +uses_types: [process_handle_go_infra, process_result_go_infra] +returns: [process_result_go_infra] +returns_optional: false +error_type: "error_go_core" +imports: [fmt, io, time] +params: + - name: handle + desc: "handle del proceso lanzado por ProcessSpawn" + - name: timeoutSec + desc: "segundos maximos de espera; 0 o negativo espera indefinidamente" +output: "resultado con exit code, stdout, stderr, duracion en ms y flag de killed" +tested: true +tests: + - "spawn and wait echo" + - "spawn with timeout kills" + - "spawn with env" + - "spawn script" + - "spawn with working dir" +test_file_path: "functions/infra/process_spawn_test.go" +file_path: "functions/infra/process_wait.go" +--- + +## Ejemplo + +```go +h, err := ProcessSpawn("sleep 60", "", nil, "") +if err != nil { + log.Fatal(err) +} +res, err := ProcessWait(h, 5) // timeout de 5 segundos +if res.Killed { + fmt.Println("proceso terminado por timeout") +} +``` + +## Notas + +Funcion impura: bloquea esperando I/O y posiblemente llama a ProcessKill. Lee stdout y stderr en goroutines separadas antes de llamar a cmd.Wait() para evitar el deadlock clasico donde cmd.Wait() bloquea porque los pipes estan llenos y nadie los lee. El exit code -1 indica que ProcessState no estaba disponible (proceso matado antes de registrar estado). diff --git a/registry.db b/registry.db index 35c24f03..59ead806 100644 Binary files a/registry.db and b/registry.db differ diff --git a/sources/sources.yaml b/sources/sources.yaml index c41a2c17..c4e3602f 100644 --- a/sources/sources.yaml +++ b/sources/sources.yaml @@ -234,3 +234,172 @@ repos: - id: wails_bind_crud_go_infra source_file: "" date: 2026-04-01 + + - repo: https://gitea-dgg044oo04woo4ggcsws4gk0.organic-machine.com/egutierrez/DevLauncher.git + license: MIT + cloned_dir: DevLauncher + extracted: + # Phase 1: Go Pure — TUI (5) + - id: apply_gradient_go_tui + source_file: launcher/core/gradient.go + date: 2026-04-08 + - id: strip_ansi_go_tui + source_file: launcher/core/commands.go + date: 2026-04-08 + - id: normalize_terminal_output_go_tui + source_file: launcher/core/commands.go + date: 2026-04-08 + - id: draw_box_go_tui + source_file: launcher/ui/styles.go + date: 2026-04-08 + - id: draw_separator_go_tui + source_file: launcher/ui/styles.go + date: 2026-04-08 + # Phase 2: Go Pure — Core (5) + - id: longest_common_prefix_go_core + source_file: launcher/core/commands.go + date: 2026-04-08 + - id: split_command_and_arg_go_core + source_file: launcher/core/commands.go + date: 2026-04-08 + - id: compare_versions_go_core + source_file: installer/core/version.go + date: 2026-04-08 + - id: parse_version_go_core + source_file: installer/core/version.go + date: 2026-04-08 + - id: rel_or_full_go_core + source_file: launcher/core/script_query.go + date: 2026-04-08 + # Phase 3: Go Impure (3) + - id: load_ascii_art_go_tui + source_file: launcher/middleware/assets.go + date: 2026-04-08 + - id: read_dir_autocomplete_go_tui + source_file: launcher/middleware/command_fs.go + date: 2026-04-08 + - id: extract_script_description_go_shell + source_file: launcher/middleware/reader.go + date: 2026-04-08 + # Phase 4: Bash Library (6) + - id: bash_colors_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + - id: bash_log_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + - id: bash_check_deps_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + - id: bash_confirm_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + - id: bash_safe_run_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + - id: bash_handle_error_bash_shell + source_file: scripts/lib/common.sh + date: 2026-04-08 + # Phase 5: Bash Cybersecurity (12) + - id: analyze_dns_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/redes/analisis_dns.sh + date: 2026-04-08 + - id: list_active_connections_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/redes/conexiones_activas.sh + date: 2026-04-08 + - id: geolocate_ip_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/redes/geoip.sh + date: 2026-04-08 + - id: audit_ssh_config_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/sistema/auditar_ssh.sh + date: 2026-04-08 + - id: check_firewall_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/sistema/firewall_status.sh + date: 2026-04-08 + - id: detect_suspicious_users_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/sistema/usuarios_sospechosos.sh + date: 2026-04-08 + - id: encrypt_file_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/utilidades/cifrar_archivo.sh + date: 2026-04-08 + - id: generate_password_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/utilidades/generar_password.sh + date: 2026-04-08 + - id: verify_file_hash_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/utilidades/verificar_hash.sh + date: 2026-04-08 + - id: audit_http_headers_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/web/cabeceras_http.sh + date: 2026-04-08 + - id: inspect_ssl_cert_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/web/ssl_cert_info.sh + date: 2026-04-08 + - id: enumerate_subdomains_bash_cybersecurity + source_file: scripts/linux/ciberseguridad/web/subdominios.sh + date: 2026-04-08 + # Phase 6: Git Utils (4) + - id: git_repo_status_bash_shell + source_file: scripts/linux/git_utils/estado_repo.sh + date: 2026-04-08 + - id: git_clean_branches_bash_shell + source_file: scripts/linux/git_utils/limpiar_ramas.sh + date: 2026-04-08 + - id: git_push_all_remotes_bash_shell + source_file: scripts/linux/git_utils/push_todos_remotes.sh + date: 2026-04-08 + - id: git_log_visual_bash_shell + source_file: scripts/linux/git_utils/historial_commits.sh + date: 2026-04-08 + # Phase 7: Infra — System (3) + - id: analyze_disk_space_bash_infra + source_file: scripts/linux/gestion_linux/espacio_disponible.sh + date: 2026-04-08 + - id: list_listening_ports_bash_infra + source_file: scripts/linux/gestion_linux/puertos_activos.sh + date: 2026-04-08 + - id: detect_wsl_bash_infra + source_file: scripts/linux/gestion_linux/wsl_host.sh + date: 2026-04-08 + # Phase 7: Infra — Installers (7) + - id: install_go_bash_infra + source_file: scripts/linux/instaladores/instalar_go.sh + date: 2026-04-08 + - id: install_nodejs_bash_infra + source_file: scripts/linux/instaladores/instalar_nodejs.sh + date: 2026-04-08 + - id: install_pnpm_bash_infra + source_file: scripts/linux/instaladores/instalar_pnpm.sh + date: 2026-04-08 + - id: install_python312_bash_infra + source_file: scripts/linux/instaladores/instalar_python312.sh + date: 2026-04-08 + - id: install_uv_bash_infra + source_file: scripts/linux/instaladores/instalar_uv.sh + date: 2026-04-08 + - id: install_volta_bash_infra + source_file: scripts/linux/instaladores/instalar_volta.sh + date: 2026-04-08 + - id: install_wails_bash_infra + source_file: scripts/linux/instaladores/instalar_wails.sh + date: 2026-04-08 + # Phase 8: Shell Utils + Init (4) + - id: convert_text_case_bash_shell + source_file: scripts/linux/conversores/conversor_texto.sh + date: 2026-04-08 + - id: create_project_structure_bash_shell + source_file: scripts/linux/inicializar_repos/functional_structure.sh + date: 2026-04-08 + - id: init_go_module_bash_pipelines + source_file: scripts/linux/inicializar_repos/go/init_go_module.sh + date: 2026-04-08 + - id: init_go_project_bash_pipelines + source_file: scripts/linux/inicializar_repos/go/init_go_proyect.sh + date: 2026-04-08 + + - repo: https://github.com/daguflow/dagu + license: GPL-3.0 + cloned_dir: dagu + analyzed: true + extracted: [] + # GPL-3.0: no code extracted. YAML format studied for compatibility in issue 0007 (dag_engine). + # Our implementation is written from scratch with no dagu code copied. diff --git a/types/core/dag_continue_on.md b/types/core/dag_continue_on.md new file mode 100644 index 00000000..df04ad4c --- /dev/null +++ b/types/core/dag_continue_on.md @@ -0,0 +1,20 @@ +--- +name: dag_continue_on +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagContinueOn struct { + Failure bool + Skipped bool + } +description: "Politica de continuacion de un step DAG ante fallos o saltos. Cuando Failure=true el DAG no se detiene si el step falla. Cuando Skipped=true tampoco se detiene si el step es saltado." +tags: [dag, workflow, policy, error-handling] +uses_types: [] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. Embebido en DagStep. Corresponde al campo `continue_on` del YAML de Dagu. diff --git a/types/core/dag_definition.md b/types/core/dag_definition.md new file mode 100644 index 00000000..21166858 --- /dev/null +++ b/types/core/dag_definition.md @@ -0,0 +1,31 @@ +--- +name: dag_definition +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagDefinition struct { + Name string + Description string + Group string + Type string + WorkingDir string + Shell string + Env map[string]string + Schedule []string + Steps []DagStep + HandlerOn DagHandlers + Tags []string + TimeoutSec int + FilePath string + } +description: "Definicion completa de un workflow DAG parseada desde YAML compatible con Dagu. Contiene steps, handlers de ciclo de vida, variables de entorno, schedule y metadatos del flujo." +tags: [dag, workflow, yaml, dagu, definition] +uses_types: [dag_step_go_core, dag_handlers_go_core] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. Type puede ser "graph" (ejecucion paralela por niveles topologicos) o vacio (cadena secuencial). Schedule es siempre una lista de strings (normalizado desde string o lista en el YAML). FilePath se rellena opcionalmente por el caller para saber el origen del archivo. diff --git a/types/core/dag_handlers.md b/types/core/dag_handlers.md new file mode 100644 index 00000000..6769d965 --- /dev/null +++ b/types/core/dag_handlers.md @@ -0,0 +1,22 @@ +--- +name: dag_handlers +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagHandlers struct { + Init []DagStep + Success []DagStep + Failure []DagStep + Exit []DagStep + } +description: "Handlers de ciclo de vida de un DAG. Cada campo contiene steps que se ejecutan en el evento correspondiente: inicializacion, exito, fallo o salida (siempre). Corresponde a handler_on o handlers en el YAML de Dagu." +tags: [dag, workflow, handlers, lifecycle] +uses_types: [dag_step_go_core] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. Embebido en DagDefinition. Los campos handler_on y handlers son aliases en el YAML de Dagu — ambos se normalizan a este tipo. Cada handler puede ser un step unico o una lista de steps en el YAML. diff --git a/types/core/dag_retry_policy.md b/types/core/dag_retry_policy.md new file mode 100644 index 00000000..f7c8e8ea --- /dev/null +++ b/types/core/dag_retry_policy.md @@ -0,0 +1,20 @@ +--- +name: dag_retry_policy +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagRetryPolicy struct { + Limit int + IntervalSec int + } +description: "Politica de reintentos automaticos para un step DAG. Limit es el numero maximo de reintentos e IntervalSec es el tiempo de espera entre intentos en segundos." +tags: [dag, workflow, retry, policy] +uses_types: [] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. Embebido en DagStep. Corresponde al campo `retry_policy` del YAML de Dagu. Limit=0 significa sin reintentos. diff --git a/types/core/dag_step.md b/types/core/dag_step.md new file mode 100644 index 00000000..1e01e66b --- /dev/null +++ b/types/core/dag_step.md @@ -0,0 +1,33 @@ +--- +name: dag_step +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagStep struct { + Name string + ID string + Description string + Command string + Script string + Args []string + Shell string + Dir string + Depends []string + Env map[string]string + ContinueOn DagContinueOn + RetryPolicy DagRetryPolicy + TimeoutSec int + Output string + Tags []string + } +description: "Un paso individual en un workflow DAG con command/script, dependencias y configuracion de reintentos. Soporta variables de entorno por step, directorio de trabajo propio y politica continue_on." +tags: [dag, workflow, step, yaml, dagu] +uses_types: [dag_continue_on_go_core, dag_retry_policy_go_core] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. El campo ID es opcional y se usa como referencia en otros steps (ej: `${id.stdout}`). Si Name e ID estan ambos vacios, el step es invalido. Env del step se mergea sobre el Env del DAG padre durante la resolucion. diff --git a/types/core/dag_validation_result.md b/types/core/dag_validation_result.md new file mode 100644 index 00000000..ee668563 --- /dev/null +++ b/types/core/dag_validation_result.md @@ -0,0 +1,22 @@ +--- +name: dag_validation_result +lang: go +domain: core +version: "1.0.0" +algebraic: product +definition: | + type DagValidationResult struct { + Valid bool + Errors []string + Warnings []string + Levels [][]string + } +description: "Resultado de validar un DagDefinition. Contiene errores estructurales (ciclos, depends invalidos, nombres duplicados), warnings (command+script simultaneos) y los niveles topologicos calculados si el DAG es valido." +tags: [dag, validation, workflow, result] +uses_types: [] +file_path: "functions/core/dag_definition.go" +--- + +## Notas + +Tipo producto. Levels solo se rellena cuando Valid=true. Cada sub-slice de Levels contiene los nombres/IDs de steps que pueden ejecutarse en paralelo. El orden de Levels refleja el orden topologico del grafo de dependencias. diff --git a/types/infra/dag_run.md b/types/infra/dag_run.md new file mode 100644 index 00000000..53ff6c64 --- /dev/null +++ b/types/infra/dag_run.md @@ -0,0 +1,27 @@ +--- +name: DagRun +lang: go +domain: infra +version: "1.0.0" +algebraic: product +definition: | + type DagRun struct { + ID string + DagName string + DagPath string + Status string + StartedAt time.Time + FinishedAt time.Time + Trigger string + Error string + } +description: "Representa una ejecucion de un workflow DAG. Almacenado en SQLite con estado, timestamps y trigger." +tags: [dag, execution, run, workflow] +uses_types: [] +file_path: "functions/infra/dag_run.go" +--- + +## Notas + +Status puede ser: pending, running, success, failed, cancelled. +Trigger puede ser: manual, cron, api. diff --git a/types/infra/dag_step_result.md b/types/infra/dag_step_result.md new file mode 100644 index 00000000..56780e5c --- /dev/null +++ b/types/infra/dag_step_result.md @@ -0,0 +1,29 @@ +--- +name: DagStepResult +lang: go +domain: infra +version: "1.0.0" +algebraic: product +definition: | + type DagStepResult struct { + ID string + RunID string + StepName string + Status string + ExitCode int + Stdout string + Stderr string + StartedAt time.Time + FinishedAt time.Time + DurationMs int64 + Error string + } +description: "Resultado de la ejecucion de un step individual dentro de un DagRun. Captura exit code, stdout, stderr y duracion." +tags: [dag, execution, step, result] +uses_types: [DagRun_go_infra] +file_path: "functions/infra/dag_run.go" +--- + +## Notas + +Status puede ser: pending, running, success, failed, skipped. diff --git a/types/infra/process_handle.md b/types/infra/process_handle.md new file mode 100644 index 00000000..83d96fbc --- /dev/null +++ b/types/infra/process_handle.md @@ -0,0 +1,24 @@ +--- +name: process_handle +lang: go +domain: infra +version: "1.0.0" +algebraic: product +definition: | + type ProcessHandle struct { + Cmd *exec.Cmd + Pid int + StartTime time.Time + Dir string + stdout io.ReadCloser + stderr io.ReadCloser + } +description: "Handle de un subproceso en ejecucion. Contiene el comando, PID, tiempo de inicio, directorio de trabajo y los pipes de stdout/stderr (privados, leidos internamente por ProcessWait)." +tags: [process, subprocess, handle, infra, exec] +uses_types: [] +file_path: "functions/infra/process_handle.go" +--- + +## Notas + +Tipo producto. Los campos stdout y stderr son privados para evitar lecturas concurrentes externas — ProcessWait los consume internamente. Cmd.SysProcAttr.Setpgid=true garantiza que ProcessKill puede matar el process group completo usando -Pid. diff --git a/types/infra/process_result.md b/types/infra/process_result.md new file mode 100644 index 00000000..4d969124 --- /dev/null +++ b/types/infra/process_result.md @@ -0,0 +1,23 @@ +--- +name: process_result +lang: go +domain: infra +version: "1.0.0" +algebraic: product +definition: | + type ProcessResult struct { + ExitCode int + Stdout string + Stderr string + DurationMs int64 + Killed bool + } +description: "Resultado de un subproceso completado. Contiene codigo de salida, salida estandar y de error, duracion en milisegundos, y un flag que indica si fue terminado por timeout." +tags: [process, subprocess, result, exit, infra, exec] +uses_types: [] +file_path: "functions/infra/process_handle.go" +--- + +## Notas + +Tipo producto — todos los campos siempre presentes. Killed=true indica que ProcessWait agoto el timeout y llamo a ProcessKill; en ese caso ExitCode suele ser -1 o el codigo de SIGKILL segun el OS. DurationMs incluye el tiempo total desde ProcessSpawn hasta que Wait() retorno.