merge: issue/0007-dag-engine — Motor de DAGs con CLI, web frontend y SQLite
Reemplaza Dagu con implementacion propia compatible con formato YAML existente. Incluye parser, validador, topo sort, process manager, execution store SQLite, scheduler cron, CLI (run/list/status/validate/server) y frontend React/Mantine.
This commit is contained in:
@@ -55,3 +55,4 @@ Thumbs.db
|
||||
|
||||
broken_paths.txt
|
||||
imgui.ini
|
||||
prompts/
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# Build output
|
||||
dag_engine
|
||||
*.exe
|
||||
|
||||
# Frontend build
|
||||
frontend/dist/
|
||||
frontend/node_modules/
|
||||
|
||||
# Go
|
||||
vendor/
|
||||
|
||||
# Editor
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// RegisterAPI sets up all HTTP routes on the given mux.
|
||||
func RegisterAPI(mux *http.ServeMux, executor *Executor, scheduler *Scheduler, frontendFS fs.FS) {
|
||||
// API routes.
|
||||
mux.HandleFunc("GET /api/dags", handleListDags(executor))
|
||||
mux.HandleFunc("GET /api/dags/{name}", handleGetDag(executor))
|
||||
mux.HandleFunc("POST /api/dags/{name}/run", handleRunDag(executor))
|
||||
|
||||
mux.HandleFunc("GET /api/runs", handleListRuns(executor))
|
||||
mux.HandleFunc("GET /api/runs/{id}", handleGetRun(executor))
|
||||
|
||||
mux.HandleFunc("POST /api/scheduler/start", handleSchedulerStart(scheduler))
|
||||
mux.HandleFunc("POST /api/scheduler/stop", handleSchedulerStop(scheduler))
|
||||
mux.HandleFunc("GET /api/scheduler/status", handleSchedulerStatus(scheduler))
|
||||
|
||||
// Frontend SPA fallback.
|
||||
if frontendFS != nil {
|
||||
mux.Handle("/", spaHandler(frontendFS))
|
||||
}
|
||||
}
|
||||
|
||||
// spaHandler serves static files from the embedded FS, falling back to index.html
|
||||
// for unknown paths (SPA client-side routing).
|
||||
func spaHandler(fsys fs.FS) http.Handler {
|
||||
fileServer := http.FileServer(http.FS(fsys))
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Try to serve the file directly.
|
||||
path := r.URL.Path
|
||||
if path == "/" {
|
||||
path = "index.html"
|
||||
} else {
|
||||
path = path[1:] // strip leading /
|
||||
}
|
||||
|
||||
if _, err := fs.Stat(fsys, path); err != nil {
|
||||
// File not found — serve index.html for SPA routing.
|
||||
r.URL.Path = "/"
|
||||
}
|
||||
fileServer.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
---
|
||||
name: dag_engine
|
||||
lang: go
|
||||
domain: infra
|
||||
description: "Motor de ejecucion de DAGs con CLI y interfaz web. Reemplaza Dagu con implementacion propia compatible con el formato YAML existente. Almacena historial de ejecuciones en SQLite."
|
||||
tags: [service, dag, workflow, scheduler, web, cron]
|
||||
uses_functions:
|
||||
- dag_parse_go_core
|
||||
- dag_validate_go_core
|
||||
- dag_topo_sort_go_core
|
||||
- dag_resolve_env_go_core
|
||||
- parse_cron_expr_go_core
|
||||
- next_cron_time_go_core
|
||||
- cron_ticker_go_infra
|
||||
- cron_match_go_core
|
||||
- process_spawn_go_infra
|
||||
- process_wait_go_infra
|
||||
- process_kill_go_infra
|
||||
uses_types:
|
||||
- dag_definition_go_core
|
||||
- dag_step_go_core
|
||||
- dag_validation_result_go_core
|
||||
- cron_schedule_go_core
|
||||
- process_handle_go_infra
|
||||
- process_result_go_infra
|
||||
- DagRun_go_infra
|
||||
- DagStepResult_go_infra
|
||||
framework: "net/http + vite + react"
|
||||
entry_point: "main.go"
|
||||
dir_path: "apps/dag_engine"
|
||||
---
|
||||
|
||||
## Arquitectura
|
||||
|
||||
CLI + servidor web en un unico binario:
|
||||
|
||||
```
|
||||
dag-engine run <path.yaml> # ejecuta un DAG desde terminal
|
||||
dag-engine list [dir] # lista DAGs con schedule y estado
|
||||
dag-engine status [dag_name] # historial de ejecuciones
|
||||
dag-engine validate <path.yaml> # valida sin ejecutar
|
||||
dag-engine server # arranca HTTP + frontend web
|
||||
```
|
||||
|
||||
### Backend (Go)
|
||||
|
||||
- `net/http` con `ServeMux` (Go 1.22+ pattern routing)
|
||||
- SQLite via `go-sqlite3` para historial de runs
|
||||
- Executor: parse -> validate -> topo_sort -> spawn/wait por nivel -> store
|
||||
- Scheduler: cron_ticker por cada DAG con schedule
|
||||
|
||||
### Frontend (Vite + React + Mantine)
|
||||
|
||||
- DagList: tabla de DAGs con schedule, tags, ultimo status
|
||||
- DagDetail: metadata + "Run Now" + historial
|
||||
- RunDetail: timeline de steps con stdout/stderr expandible
|
||||
|
||||
### Storage
|
||||
|
||||
SQLite `dag_engine.db`:
|
||||
- `dag_runs`: id, dag_name, status, trigger, started_at, finished_at, error
|
||||
- `dag_step_results`: id, run_id, step_name, status, exit_code, stdout, stderr, duration_ms
|
||||
|
||||
### Build
|
||||
|
||||
```bash
|
||||
cd frontend && pnpm install && pnpm build
|
||||
cd .. && CGO_ENABLED=1 go build -tags fts5 -o dag-engine .
|
||||
```
|
||||
|
||||
### Uso
|
||||
|
||||
```bash
|
||||
# CLI
|
||||
./dag-engine run ~/dagu/dags/example.yaml
|
||||
./dag-engine list ~/dagu/dags/
|
||||
|
||||
# Servidor web
|
||||
./dag-engine server --port 8090 --dags-dir ~/dagu/dags/ --scheduler
|
||||
# Browser: http://localhost:8090
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Compatible con el formato YAML de Dagu. Lee DAGs existentes de `~/dagu/dags/` sin modificaciones.
|
||||
Puerto por defecto 8090 (mismo que Dagu).
|
||||
@@ -0,0 +1,34 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// Config holds the runtime configuration for the DAG engine.
|
||||
type Config struct {
|
||||
Port int
|
||||
DagsDir string
|
||||
DBPath string
|
||||
AutoScheduler bool
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
func DefaultConfig() Config {
|
||||
home, _ := os.UserHomeDir()
|
||||
return Config{
|
||||
Port: 8090,
|
||||
DagsDir: filepath.Join(home, "dagu", "dags"),
|
||||
DBPath: "dag_engine.db",
|
||||
}
|
||||
}
|
||||
|
||||
// ParseFlags populates config from CLI flags for the "server" subcommand.
|
||||
func (c *Config) ParseFlags(fs *flag.FlagSet, args []string) error {
|
||||
fs.IntVar(&c.Port, "port", c.Port, "HTTP server port")
|
||||
fs.StringVar(&c.DagsDir, "dags-dir", c.DagsDir, "directory containing DAG YAML files")
|
||||
fs.StringVar(&c.DBPath, "db", c.DBPath, "path to SQLite database")
|
||||
fs.BoolVar(&c.AutoScheduler, "scheduler", c.AutoScheduler, "auto-start cron scheduler")
|
||||
return fs.Parse(args)
|
||||
}
|
||||
@@ -0,0 +1,482 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"fn-registry/functions/core"
|
||||
"fn-registry/functions/infra"
|
||||
|
||||
"dag-engine/store"
|
||||
)
|
||||
|
||||
// Executor orchestrates DAG parsing, validation, and execution.
|
||||
type Executor struct {
|
||||
store *store.DB
|
||||
dagsDir string
|
||||
}
|
||||
|
||||
// NewExecutor creates a new executor.
|
||||
func NewExecutor(s *store.DB, dagsDir string) *Executor {
|
||||
return &Executor{store: s, dagsDir: dagsDir}
|
||||
}
|
||||
|
||||
// ExecuteDAG runs a DAG from a YAML file path and returns the run ID.
|
||||
// It runs asynchronously: steps execute in topological order with parallel levels.
|
||||
func (e *Executor) ExecuteDAG(ctx context.Context, dagPath string, trigger string) (string, error) {
|
||||
data, err := os.ReadFile(dagPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read dag: %w", err)
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse dag: %w", err)
|
||||
}
|
||||
dag.FilePath = dagPath
|
||||
|
||||
// Resolve env variables.
|
||||
dag = core.DagResolveEnv(dag, os.Environ())
|
||||
|
||||
// Validate.
|
||||
result := core.DagValidate(dag)
|
||||
if !result.Valid {
|
||||
return "", fmt.Errorf("validate dag: %s", strings.Join(result.Errors, "; "))
|
||||
}
|
||||
|
||||
// Create run record.
|
||||
runID := generateID()
|
||||
now := time.Now()
|
||||
run := &store.DagRun{
|
||||
ID: runID,
|
||||
DagName: dag.Name,
|
||||
DagPath: dagPath,
|
||||
Status: "running",
|
||||
Trigger: trigger,
|
||||
StartedAt: now,
|
||||
}
|
||||
if err := e.store.CreateRun(run); err != nil {
|
||||
return "", fmt.Errorf("create run: %w", err)
|
||||
}
|
||||
|
||||
// Topological sort.
|
||||
levels, err := core.DagTopoSort(dag.Steps)
|
||||
if err != nil {
|
||||
e.failRun(runID, err)
|
||||
return runID, err
|
||||
}
|
||||
|
||||
// Setup DAGU_ENV temp file for inter-step communication.
|
||||
daguEnvFile, err := os.CreateTemp("", "dagu_env_*")
|
||||
if err != nil {
|
||||
e.failRun(runID, err)
|
||||
return runID, err
|
||||
}
|
||||
daguEnvPath := daguEnvFile.Name()
|
||||
daguEnvFile.Close()
|
||||
defer os.Remove(daguEnvPath)
|
||||
|
||||
// Track step outputs for ${step_id.stdout} references.
|
||||
stepOutputs := make(map[string]string)
|
||||
|
||||
// Execute levels.
|
||||
runFailed := false
|
||||
var runErr error
|
||||
|
||||
for _, level := range levels {
|
||||
if runFailed {
|
||||
// Skip remaining levels, mark steps as skipped.
|
||||
for _, step := range level {
|
||||
e.recordStepSkipped(runID, step)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
levelFailed := false
|
||||
|
||||
for _, step := range level {
|
||||
step := step
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
mu.Lock()
|
||||
if levelFailed {
|
||||
mu.Unlock()
|
||||
e.recordStepSkipped(runID, step)
|
||||
return
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
err := e.executeStep(ctx, runID, dag, step, daguEnvPath, stepOutputs, &mu)
|
||||
if err != nil && !step.ContinueOn.Failure {
|
||||
mu.Lock()
|
||||
levelFailed = true
|
||||
runFailed = true
|
||||
runErr = fmt.Errorf("step %q failed: %w", stepName(step), err)
|
||||
mu.Unlock()
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// Run handlers.
|
||||
if runFailed {
|
||||
e.runHandlers(ctx, runID, dag, dag.HandlerOn.Failure, daguEnvPath, stepOutputs)
|
||||
} else {
|
||||
e.runHandlers(ctx, runID, dag, dag.HandlerOn.Success, daguEnvPath, stepOutputs)
|
||||
}
|
||||
e.runHandlers(ctx, runID, dag, dag.HandlerOn.Exit, daguEnvPath, stepOutputs)
|
||||
|
||||
// Finalize run.
|
||||
fin := time.Now()
|
||||
status := "success"
|
||||
errMsg := ""
|
||||
if runFailed {
|
||||
status = "failed"
|
||||
if runErr != nil {
|
||||
errMsg = runErr.Error()
|
||||
}
|
||||
}
|
||||
e.store.UpdateRunStatus(runID, status, &fin, errMsg)
|
||||
|
||||
return runID, runErr
|
||||
}
|
||||
|
||||
// executeStep runs a single step, recording results in the store.
|
||||
func (e *Executor) executeStep(ctx context.Context, runID string, dag core.DagDefinition, step core.DagStep, daguEnvPath string, outputs map[string]string, mu *sync.Mutex) error {
|
||||
stepID := generateID()
|
||||
now := time.Now()
|
||||
e.store.InsertStepResult(&store.DagStepResult{
|
||||
ID: stepID,
|
||||
RunID: runID,
|
||||
StepName: stepName(step),
|
||||
Status: "running",
|
||||
StartedAt: &now,
|
||||
})
|
||||
|
||||
// Build environment.
|
||||
env := buildStepEnv(dag, step, daguEnvPath, outputs)
|
||||
|
||||
// Determine command.
|
||||
command := step.Command
|
||||
if command == "" && step.Script != "" {
|
||||
command = step.Script
|
||||
}
|
||||
if command == "" {
|
||||
e.store.UpdateStepResult(stepID, "skipped", 0, "", "", nil, 0, "no command or script")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Resolve step-level ${VAR} references and ${step_id.stdout} patterns.
|
||||
mu.Lock()
|
||||
command = resolveStepRefs(command, outputs)
|
||||
mu.Unlock()
|
||||
|
||||
// Determine working directory.
|
||||
dir := step.Dir
|
||||
if dir == "" {
|
||||
dir = dag.WorkingDir
|
||||
}
|
||||
|
||||
shell := step.Shell
|
||||
if shell == "" {
|
||||
shell = dag.Shell
|
||||
}
|
||||
|
||||
// Spawn process.
|
||||
handle, err := infra.ProcessSpawn(command, dir, env, shell)
|
||||
if err != nil {
|
||||
fin := time.Now()
|
||||
e.store.UpdateStepResult(stepID, "failed", -1, "", "", &fin, time.Since(now).Milliseconds(), err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Wait for process.
|
||||
result, err := infra.ProcessWait(handle, step.TimeoutSec)
|
||||
fin := time.Now()
|
||||
duration := time.Since(now).Milliseconds()
|
||||
|
||||
if err != nil && result.ExitCode == 0 {
|
||||
result.ExitCode = -1
|
||||
}
|
||||
|
||||
status := "success"
|
||||
errMsg := ""
|
||||
if result.ExitCode != 0 || err != nil {
|
||||
status = "failed"
|
||||
if err != nil {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
}
|
||||
|
||||
e.store.UpdateStepResult(stepID, status, result.ExitCode, result.Stdout, result.Stderr, &fin, duration, errMsg)
|
||||
|
||||
// Store output for ${step_id.stdout} references.
|
||||
if step.ID != "" || step.Output != "" {
|
||||
mu.Lock()
|
||||
key := step.ID
|
||||
if key == "" {
|
||||
key = step.Output
|
||||
}
|
||||
outputs[key] = strings.TrimSpace(result.Stdout)
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
// Read DAGU_ENV for inter-step env propagation.
|
||||
readDaguEnv(daguEnvPath, outputs)
|
||||
|
||||
if status == "failed" {
|
||||
return fmt.Errorf("exit code %d", result.ExitCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Executor) runHandlers(ctx context.Context, runID string, dag core.DagDefinition, handlers []core.DagStep, daguEnvPath string, outputs map[string]string) {
|
||||
var mu sync.Mutex
|
||||
for _, step := range handlers {
|
||||
e.executeStep(ctx, runID, dag, step, daguEnvPath, outputs, &mu)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Executor) failRun(runID string, err error) {
|
||||
fin := time.Now()
|
||||
e.store.UpdateRunStatus(runID, "failed", &fin, err.Error())
|
||||
}
|
||||
|
||||
func (e *Executor) recordStepSkipped(runID string, step core.DagStep) {
|
||||
now := time.Now()
|
||||
e.store.InsertStepResult(&store.DagStepResult{
|
||||
ID: generateID(),
|
||||
RunID: runID,
|
||||
StepName: stepName(step),
|
||||
Status: "skipped",
|
||||
StartedAt: &now,
|
||||
})
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func stepName(s core.DagStep) string {
|
||||
if s.Name != "" {
|
||||
return s.Name
|
||||
}
|
||||
return s.ID
|
||||
}
|
||||
|
||||
func buildStepEnv(dag core.DagDefinition, step core.DagStep, daguEnvPath string, outputs map[string]string) []string {
|
||||
env := os.Environ()
|
||||
|
||||
// Add DAG-level env.
|
||||
for k, v := range dag.Env {
|
||||
env = append(env, k+"="+v)
|
||||
}
|
||||
|
||||
// Add step-level env.
|
||||
for k, v := range step.Env {
|
||||
env = append(env, k+"="+v)
|
||||
}
|
||||
|
||||
// Add DAGU_ENV path.
|
||||
env = append(env, "DAGU_ENV="+daguEnvPath)
|
||||
|
||||
return env
|
||||
}
|
||||
|
||||
func resolveStepRefs(command string, outputs map[string]string) string {
|
||||
for k, v := range outputs {
|
||||
command = strings.ReplaceAll(command, "${"+k+".stdout}", v)
|
||||
command = strings.ReplaceAll(command, "$"+k+".stdout", v)
|
||||
}
|
||||
return command
|
||||
}
|
||||
|
||||
func readDaguEnv(path string, outputs map[string]string) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil || len(data) == 0 {
|
||||
return
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, "=", 2)
|
||||
if len(parts) == 2 {
|
||||
outputs[parts[0]] = parts[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateID creates a simple time-based unique ID.
|
||||
func generateID() string {
|
||||
return fmt.Sprintf("%d-%04x", time.Now().UnixNano(), time.Now().Nanosecond()%0xFFFF)
|
||||
}
|
||||
|
||||
// --- DAG listing helpers ---
|
||||
|
||||
// DagInfo summarizes a DAG file for listing.
|
||||
type DagInfo struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Schedule []string `json:"schedule,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Type string `json:"type,omitempty"`
|
||||
FilePath string `json:"file_path"`
|
||||
Valid bool `json:"valid"`
|
||||
LastRun *store.DagRun `json:"last_run,omitempty"`
|
||||
}
|
||||
|
||||
// ListDAGs scans a directory for YAML files and returns parsed DAG info.
|
||||
func (e *Executor) ListDAGs() ([]DagInfo, error) {
|
||||
entries, err := os.ReadDir(e.dagsDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read dags dir: %w", err)
|
||||
}
|
||||
|
||||
var dags []DagInfo
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
ext := filepath.Ext(entry.Name())
|
||||
if ext != ".yaml" && ext != ".yml" {
|
||||
continue
|
||||
}
|
||||
|
||||
path := filepath.Join(e.dagsDir, entry.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
dags = append(dags, DagInfo{
|
||||
Name: strings.TrimSuffix(entry.Name(), ext),
|
||||
FilePath: path,
|
||||
Valid: false,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
info := DagInfo{
|
||||
Name: dag.Name,
|
||||
Description: dag.Description,
|
||||
Schedule: dag.Schedule,
|
||||
Tags: dag.Tags,
|
||||
Type: dag.Type,
|
||||
FilePath: path,
|
||||
Valid: true,
|
||||
}
|
||||
|
||||
// Attach last run info.
|
||||
runs, _, _ := e.store.ListRuns(dag.Name, 1, 0)
|
||||
if len(runs) > 0 {
|
||||
info.LastRun = &runs[0]
|
||||
}
|
||||
|
||||
dags = append(dags, info)
|
||||
}
|
||||
|
||||
return dags, nil
|
||||
}
|
||||
|
||||
// GetDAG returns detailed info for a specific DAG by name.
|
||||
func (e *Executor) GetDAG(name string) (*DagInfo, *core.DagDefinition, *core.DagValidationResult, error) {
|
||||
// Find the YAML file.
|
||||
entries, err := os.ReadDir(e.dagsDir)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
ext := filepath.Ext(entry.Name())
|
||||
base := strings.TrimSuffix(entry.Name(), ext)
|
||||
if (ext != ".yaml" && ext != ".yml") || base != name {
|
||||
continue
|
||||
}
|
||||
|
||||
path := filepath.Join(e.dagsDir, entry.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("parse: %w", err)
|
||||
}
|
||||
dag.FilePath = path
|
||||
|
||||
validationResult := core.DagValidate(dag)
|
||||
|
||||
info := &DagInfo{
|
||||
Name: dag.Name,
|
||||
Description: dag.Description,
|
||||
Schedule: dag.Schedule,
|
||||
Tags: dag.Tags,
|
||||
Type: dag.Type,
|
||||
FilePath: path,
|
||||
Valid: validationResult.Valid,
|
||||
}
|
||||
|
||||
runs, _, _ := e.store.ListRuns(dag.Name, 1, 0)
|
||||
if len(runs) > 0 {
|
||||
info.LastRun = &runs[0]
|
||||
}
|
||||
|
||||
return info, &dag, &validationResult, nil
|
||||
}
|
||||
|
||||
return nil, nil, nil, fmt.Errorf("dag %q not found in %s", name, e.dagsDir)
|
||||
}
|
||||
|
||||
// ValidateDAG parses and validates a DAG file, printing results.
|
||||
func ValidateDAG(path string) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse error: %w", err)
|
||||
}
|
||||
|
||||
result := core.DagValidate(dag)
|
||||
|
||||
log.Printf("DAG: %s", dag.Name)
|
||||
log.Printf("Steps: %d", len(dag.Steps))
|
||||
log.Printf("Schedule: %v", dag.Schedule)
|
||||
|
||||
if result.Valid {
|
||||
log.Printf("Validation: PASS")
|
||||
log.Printf("Topological levels: %d", len(result.Levels))
|
||||
for i, level := range result.Levels {
|
||||
log.Printf(" Level %d: %v", i, level)
|
||||
}
|
||||
} else {
|
||||
log.Printf("Validation: FAIL")
|
||||
for _, e := range result.Errors {
|
||||
log.Printf(" ERROR: %s", e)
|
||||
}
|
||||
}
|
||||
for _, w := range result.Warnings {
|
||||
log.Printf(" WARNING: %s", w)
|
||||
}
|
||||
|
||||
if !result.Valid {
|
||||
return fmt.Errorf("validation failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>DAG Engine</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "dag-engine-frontend",
|
||||
"private": true,
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"preview": "vite preview"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mantine/core": "^9.0.2",
|
||||
"@mantine/hooks": "^9.0.2",
|
||||
"@tabler/icons-react": "^3.31.0",
|
||||
"react": "^19.1.0",
|
||||
"react-dom": "^19.1.0",
|
||||
"react-router-dom": "^7.6.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.1.6",
|
||||
"@types/react-dom": "^19.1.6",
|
||||
"@vitejs/plugin-react": "^4.5.2",
|
||||
"postcss": "^8.5.4",
|
||||
"postcss-preset-mantine": "^1.17.0",
|
||||
"typescript": "~5.8.3",
|
||||
"vite": "^6.3.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module.exports = {
|
||||
plugins: {
|
||||
"postcss-preset-mantine": {},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,32 @@
|
||||
import { Routes, Route } from "react-router-dom";
|
||||
import { AppShell, Container, Title, Group, Text } from "@mantine/core";
|
||||
import { IconTopologyRing } from "@tabler/icons-react";
|
||||
import { DagList } from "./pages/DagList";
|
||||
import { DagDetail } from "./pages/DagDetail";
|
||||
import { RunDetail } from "./pages/RunDetail";
|
||||
|
||||
export function App() {
|
||||
return (
|
||||
<AppShell header={{ height: 50 }} padding="md">
|
||||
<AppShell.Header>
|
||||
<Group h="100%" px="md">
|
||||
<IconTopologyRing size={24} />
|
||||
<Title order={4}>DAG Engine</Title>
|
||||
<Text size="xs" c="dimmed">
|
||||
fn_registry workflow executor
|
||||
</Text>
|
||||
</Group>
|
||||
</AppShell.Header>
|
||||
|
||||
<AppShell.Main>
|
||||
<Container size="lg">
|
||||
<Routes>
|
||||
<Route path="/" element={<DagList />} />
|
||||
<Route path="/dags/:name" element={<DagDetail />} />
|
||||
<Route path="/runs/:id" element={<RunDetail />} />
|
||||
</Routes>
|
||||
</Container>
|
||||
</AppShell.Main>
|
||||
</AppShell>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import type {
|
||||
DagSummary,
|
||||
DagDetail,
|
||||
DagRun,
|
||||
RunDetail,
|
||||
SchedulerStatus,
|
||||
} from "./types";
|
||||
|
||||
const BASE = "/api";
|
||||
|
||||
async function fetchJSON<T>(path: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(`${BASE}${path}`, init);
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: res.statusText }));
|
||||
throw new Error(err.error || res.statusText);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
export function listDags(): Promise<DagSummary[]> {
|
||||
return fetchJSON("/dags");
|
||||
}
|
||||
|
||||
export function getDag(name: string): Promise<DagDetail> {
|
||||
return fetchJSON(`/dags/${encodeURIComponent(name)}`);
|
||||
}
|
||||
|
||||
export function triggerDag(
|
||||
name: string
|
||||
): Promise<{ status: string; dag: string; message: string }> {
|
||||
return fetchJSON(`/dags/${encodeURIComponent(name)}/run`, {
|
||||
method: "POST",
|
||||
});
|
||||
}
|
||||
|
||||
export function listRuns(params?: {
|
||||
dag?: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}): Promise<{ runs: DagRun[]; total: number }> {
|
||||
const search = new URLSearchParams();
|
||||
if (params?.dag) search.set("dag", params.dag);
|
||||
if (params?.limit) search.set("limit", String(params.limit));
|
||||
if (params?.offset) search.set("offset", String(params.offset));
|
||||
const qs = search.toString();
|
||||
return fetchJSON(`/runs${qs ? "?" + qs : ""}`);
|
||||
}
|
||||
|
||||
export function getRun(id: string): Promise<RunDetail> {
|
||||
return fetchJSON(`/runs/${encodeURIComponent(id)}`);
|
||||
}
|
||||
|
||||
export function startScheduler(): Promise<void> {
|
||||
return fetchJSON("/scheduler/start", { method: "POST" });
|
||||
}
|
||||
|
||||
export function stopScheduler(): Promise<void> {
|
||||
return fetchJSON("/scheduler/stop", { method: "POST" });
|
||||
}
|
||||
|
||||
export function getSchedulerStatus(): Promise<SchedulerStatus> {
|
||||
return fetchJSON("/scheduler/status");
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Badge } from "@mantine/core";
|
||||
|
||||
const colorMap: Record<string, string> = {
|
||||
success: "green",
|
||||
failed: "red",
|
||||
running: "blue",
|
||||
pending: "gray",
|
||||
cancelled: "yellow",
|
||||
skipped: "dimmed",
|
||||
};
|
||||
|
||||
export function StatusBadge({ status }: { status: string }) {
|
||||
return (
|
||||
<Badge color={colorMap[status] || "gray"} variant="light" size="sm">
|
||||
{status}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import { Timeline, Text, Code, Collapse, Box, Group } from "@mantine/core";
|
||||
import {
|
||||
IconCircleCheck,
|
||||
IconCircleX,
|
||||
IconLoader,
|
||||
IconCircleMinus,
|
||||
IconClock,
|
||||
} from "@tabler/icons-react";
|
||||
import { useDisclosure } from "@mantine/hooks";
|
||||
import type { DagStepResult } from "../types";
|
||||
|
||||
const iconMap: Record<string, React.ReactNode> = {
|
||||
success: <IconCircleCheck size={16} color="var(--mantine-color-green-6)" />,
|
||||
failed: <IconCircleX size={16} color="var(--mantine-color-red-6)" />,
|
||||
running: <IconLoader size={16} color="var(--mantine-color-blue-6)" />,
|
||||
skipped: <IconCircleMinus size={16} color="var(--mantine-color-dimmed)" />,
|
||||
pending: <IconClock size={16} color="var(--mantine-color-gray-6)" />,
|
||||
};
|
||||
|
||||
function StepItem({ step }: { step: DagStepResult }) {
|
||||
const [opened, { toggle }] = useDisclosure(step.Status === "failed");
|
||||
const hasOutput = step.Stdout || step.Stderr;
|
||||
|
||||
return (
|
||||
<Timeline.Item
|
||||
bullet={iconMap[step.Status] || iconMap.pending}
|
||||
title={
|
||||
<Group gap="xs">
|
||||
<Text
|
||||
size="sm"
|
||||
fw={500}
|
||||
onClick={hasOutput ? toggle : undefined}
|
||||
style={hasOutput ? { cursor: "pointer" } : undefined}
|
||||
>
|
||||
{step.StepName}
|
||||
</Text>
|
||||
<Text size="xs" c="dimmed">
|
||||
{step.DurationMs}ms
|
||||
</Text>
|
||||
{step.ExitCode !== 0 && step.ExitCode !== -1 && (
|
||||
<Text size="xs" c="red">
|
||||
exit {step.ExitCode}
|
||||
</Text>
|
||||
)}
|
||||
</Group>
|
||||
}
|
||||
>
|
||||
{hasOutput && (
|
||||
<Collapse in={opened}>
|
||||
<Box mt="xs">
|
||||
{step.Stdout && (
|
||||
<Code block mb="xs" style={{ maxHeight: 200, overflow: "auto" }}>
|
||||
{step.Stdout}
|
||||
</Code>
|
||||
)}
|
||||
{step.Stderr && (
|
||||
<Code
|
||||
block
|
||||
color="red"
|
||||
style={{ maxHeight: 200, overflow: "auto" }}
|
||||
>
|
||||
{step.Stderr}
|
||||
</Code>
|
||||
)}
|
||||
</Box>
|
||||
</Collapse>
|
||||
)}
|
||||
</Timeline.Item>
|
||||
);
|
||||
}
|
||||
|
||||
export function StepTimeline({ steps }: { steps: DagStepResult[] }) {
|
||||
const activeIndex = steps.findIndex((s) => s.Status === "running");
|
||||
|
||||
return (
|
||||
<Timeline
|
||||
active={activeIndex >= 0 ? activeIndex : steps.length - 1}
|
||||
bulletSize={24}
|
||||
>
|
||||
{steps.map((step) => (
|
||||
<StepItem key={step.ID} step={step} />
|
||||
))}
|
||||
</Timeline>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import "@mantine/core/styles.css";
|
||||
import { MantineProvider, createTheme } from "@mantine/core";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { BrowserRouter } from "react-router-dom";
|
||||
import { App } from "./App";
|
||||
|
||||
const theme = createTheme({
|
||||
primaryColor: "blue",
|
||||
fontFamily: "system-ui, -apple-system, sans-serif",
|
||||
});
|
||||
|
||||
createRoot(document.getElementById("root")!).render(
|
||||
<MantineProvider theme={theme} defaultColorScheme="dark">
|
||||
<BrowserRouter>
|
||||
<App />
|
||||
</BrowserRouter>
|
||||
</MantineProvider>
|
||||
);
|
||||
@@ -0,0 +1,204 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useParams, useNavigate } from "react-router-dom";
|
||||
import {
|
||||
Title,
|
||||
Text,
|
||||
Group,
|
||||
Button,
|
||||
Badge,
|
||||
Stack,
|
||||
Paper,
|
||||
Table,
|
||||
Alert,
|
||||
Loader,
|
||||
Code,
|
||||
} from "@mantine/core";
|
||||
import { IconPlayerPlay, IconArrowLeft } from "@tabler/icons-react";
|
||||
import { getDag, triggerDag } from "../api";
|
||||
import { StatusBadge } from "../components/StatusBadge";
|
||||
import type { DagDetail as DagDetailType } from "../types";
|
||||
|
||||
export function DagDetail() {
|
||||
const { name } = useParams<{ name: string }>();
|
||||
const navigate = useNavigate();
|
||||
const [data, setData] = useState<DagDetailType | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [triggering, setTriggering] = useState(false);
|
||||
|
||||
const load = async () => {
|
||||
if (!name) return;
|
||||
setLoading(true);
|
||||
try {
|
||||
setData(await getDag(name));
|
||||
setError(null);
|
||||
} catch (e) {
|
||||
setError((e as Error).message);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
}, [name]);
|
||||
|
||||
const handleRun = async () => {
|
||||
if (!name) return;
|
||||
setTriggering(true);
|
||||
try {
|
||||
await triggerDag(name);
|
||||
setTimeout(load, 1000);
|
||||
} catch (e) {
|
||||
setError((e as Error).message);
|
||||
} finally {
|
||||
setTriggering(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) return <Loader />;
|
||||
if (error) return <Alert color="red">{error}</Alert>;
|
||||
if (!data) return <Text>Not found</Text>;
|
||||
|
||||
const { dag, validation, runs } = data;
|
||||
|
||||
return (
|
||||
<Stack gap="md">
|
||||
<Group>
|
||||
<Button
|
||||
variant="subtle"
|
||||
size="xs"
|
||||
leftSection={<IconArrowLeft size={14} />}
|
||||
onClick={() => navigate("/")}
|
||||
>
|
||||
Back
|
||||
</Button>
|
||||
</Group>
|
||||
|
||||
<Group justify="space-between">
|
||||
<div>
|
||||
<Title order={2}>{dag.Name}</Title>
|
||||
{dag.Description && (
|
||||
<Text size="sm" c="dimmed">
|
||||
{dag.Description}
|
||||
</Text>
|
||||
)}
|
||||
</div>
|
||||
<Button
|
||||
leftSection={<IconPlayerPlay size={16} />}
|
||||
onClick={handleRun}
|
||||
loading={triggering}
|
||||
>
|
||||
Run Now
|
||||
</Button>
|
||||
</Group>
|
||||
|
||||
<Group gap="xs">
|
||||
{dag.Schedule?.map((s: string) => (
|
||||
<Badge key={s} variant="light" ff="monospace">
|
||||
{s}
|
||||
</Badge>
|
||||
))}
|
||||
<Badge variant="light">{dag.Type || "chain"}</Badge>
|
||||
{dag.Tags?.map((t: string) => (
|
||||
<Badge key={t} variant="dot">
|
||||
{t}
|
||||
</Badge>
|
||||
))}
|
||||
</Group>
|
||||
|
||||
{!validation.Valid && (
|
||||
<Alert color="red" title="Validation errors">
|
||||
{validation.Errors.map((e: string, i: number) => (
|
||||
<Text key={i} size="sm">
|
||||
{e}
|
||||
</Text>
|
||||
))}
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
<Paper p="md" withBorder>
|
||||
<Title order={4} mb="sm">
|
||||
Steps ({dag.Steps?.length || 0})
|
||||
</Title>
|
||||
{validation.Levels?.map((level: string[], i: number) => (
|
||||
<Group key={i} gap="xs" mb="xs">
|
||||
<Text size="xs" c="dimmed" w={60}>
|
||||
Level {i}:
|
||||
</Text>
|
||||
{level.map((name: string) => {
|
||||
const step = dag.Steps?.find(
|
||||
(s) => s.Name === name || s.ID === name
|
||||
);
|
||||
return (
|
||||
<Badge key={name} variant="outline" size="sm">
|
||||
{name}
|
||||
{step?.Depends?.length
|
||||
? ` (after ${step.Depends.join(",")})`
|
||||
: ""}
|
||||
</Badge>
|
||||
);
|
||||
})}
|
||||
</Group>
|
||||
))}
|
||||
|
||||
{dag.Env && Object.keys(dag.Env).length > 0 && (
|
||||
<>
|
||||
<Title order={5} mt="md" mb="xs">
|
||||
Environment
|
||||
</Title>
|
||||
<Code block>
|
||||
{Object.entries(dag.Env)
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join("\n")}
|
||||
</Code>
|
||||
</>
|
||||
)}
|
||||
</Paper>
|
||||
|
||||
<Paper p="md" withBorder>
|
||||
<Title order={4} mb="sm">
|
||||
Run History
|
||||
</Title>
|
||||
{runs?.length ? (
|
||||
<Table striped>
|
||||
<Table.Thead>
|
||||
<Table.Tr>
|
||||
<Table.Th>Status</Table.Th>
|
||||
<Table.Th>Trigger</Table.Th>
|
||||
<Table.Th>Started</Table.Th>
|
||||
<Table.Th>Duration</Table.Th>
|
||||
</Table.Tr>
|
||||
</Table.Thead>
|
||||
<Table.Tbody>
|
||||
{runs.map((r) => (
|
||||
<Table.Tr
|
||||
key={r.ID}
|
||||
style={{ cursor: "pointer" }}
|
||||
onClick={() => navigate(`/runs/${r.ID}`)}
|
||||
>
|
||||
<Table.Td>
|
||||
<StatusBadge status={r.Status} />
|
||||
</Table.Td>
|
||||
<Table.Td>{r.Trigger}</Table.Td>
|
||||
<Table.Td>
|
||||
{new Date(r.StartedAt).toLocaleString()}
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
{r.FinishedAt
|
||||
? `${Math.round((new Date(r.FinishedAt).getTime() - new Date(r.StartedAt).getTime()) / 1000)}s`
|
||||
: "running..."}
|
||||
</Table.Td>
|
||||
</Table.Tr>
|
||||
))}
|
||||
</Table.Tbody>
|
||||
</Table>
|
||||
) : (
|
||||
<Text size="sm" c="dimmed">
|
||||
No runs yet
|
||||
</Text>
|
||||
)}
|
||||
</Paper>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import {
|
||||
Table,
|
||||
Title,
|
||||
Group,
|
||||
Button,
|
||||
Badge,
|
||||
Text,
|
||||
Loader,
|
||||
Stack,
|
||||
Alert,
|
||||
} from "@mantine/core";
|
||||
import {
|
||||
IconPlayerPlay,
|
||||
IconPlayerStop,
|
||||
IconRefresh,
|
||||
} from "@tabler/icons-react";
|
||||
import { listDags, getSchedulerStatus, startScheduler, stopScheduler } from "../api";
|
||||
import { StatusBadge } from "../components/StatusBadge";
|
||||
import type { DagSummary, SchedulerStatus } from "../types";
|
||||
|
||||
export function DagList() {
|
||||
const [dags, setDags] = useState<DagSummary[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [scheduler, setScheduler] = useState<SchedulerStatus | null>(null);
|
||||
const navigate = useNavigate();
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const [d, s] = await Promise.all([listDags(), getSchedulerStatus()]);
|
||||
setDags(d || []);
|
||||
setScheduler(s);
|
||||
} catch (e) {
|
||||
setError((e as Error).message);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
const interval = setInterval(load, 10000);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
const toggleScheduler = async () => {
|
||||
if (scheduler?.running) {
|
||||
await stopScheduler();
|
||||
} else {
|
||||
await startScheduler();
|
||||
}
|
||||
const s = await getSchedulerStatus();
|
||||
setScheduler(s);
|
||||
};
|
||||
|
||||
return (
|
||||
<Stack gap="md">
|
||||
<Group justify="space-between">
|
||||
<Title order={2}>DAGs</Title>
|
||||
<Group gap="xs">
|
||||
<Button
|
||||
size="xs"
|
||||
variant="light"
|
||||
leftSection={<IconRefresh size={14} />}
|
||||
onClick={load}
|
||||
>
|
||||
Refresh
|
||||
</Button>
|
||||
<Button
|
||||
size="xs"
|
||||
variant={scheduler?.running ? "filled" : "light"}
|
||||
color={scheduler?.running ? "green" : "gray"}
|
||||
leftSection={
|
||||
scheduler?.running ? (
|
||||
<IconPlayerStop size={14} />
|
||||
) : (
|
||||
<IconPlayerPlay size={14} />
|
||||
)
|
||||
}
|
||||
onClick={toggleScheduler}
|
||||
>
|
||||
Scheduler {scheduler?.running ? "ON" : "OFF"}
|
||||
</Button>
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
{error && <Alert color="red">{error}</Alert>}
|
||||
|
||||
{loading && !dags.length ? (
|
||||
<Loader />
|
||||
) : (
|
||||
<Table striped highlightOnHover>
|
||||
<Table.Thead>
|
||||
<Table.Tr>
|
||||
<Table.Th>Name</Table.Th>
|
||||
<Table.Th>Schedule</Table.Th>
|
||||
<Table.Th>Type</Table.Th>
|
||||
<Table.Th>Tags</Table.Th>
|
||||
<Table.Th>Last Status</Table.Th>
|
||||
<Table.Th>Last Run</Table.Th>
|
||||
</Table.Tr>
|
||||
</Table.Thead>
|
||||
<Table.Tbody>
|
||||
{dags.map((d) => (
|
||||
<Table.Tr
|
||||
key={d.file_path}
|
||||
style={{ cursor: "pointer" }}
|
||||
onClick={() => navigate(`/dags/${d.name}`)}
|
||||
>
|
||||
<Table.Td>
|
||||
<Text fw={500}>{d.name}</Text>
|
||||
{d.description && (
|
||||
<Text size="xs" c="dimmed" lineClamp={1}>
|
||||
{d.description}
|
||||
</Text>
|
||||
)}
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
<Text size="xs" ff="monospace">
|
||||
{d.schedule?.join(", ") || "-"}
|
||||
</Text>
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
<Badge variant="light" size="xs">
|
||||
{d.type || "chain"}
|
||||
</Badge>
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
<Group gap={4}>
|
||||
{d.tags?.map((t) => (
|
||||
<Badge key={t} variant="dot" size="xs">
|
||||
{t}
|
||||
</Badge>
|
||||
))}
|
||||
</Group>
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
{d.last_run ? (
|
||||
<StatusBadge status={d.last_run.Status} />
|
||||
) : (
|
||||
<Text size="xs" c="dimmed">
|
||||
-
|
||||
</Text>
|
||||
)}
|
||||
</Table.Td>
|
||||
<Table.Td>
|
||||
<Text size="xs">
|
||||
{d.last_run
|
||||
? new Date(d.last_run.StartedAt).toLocaleString()
|
||||
: "-"}
|
||||
</Text>
|
||||
</Table.Td>
|
||||
</Table.Tr>
|
||||
))}
|
||||
</Table.Tbody>
|
||||
</Table>
|
||||
)}
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useParams, useNavigate } from "react-router-dom";
|
||||
import {
|
||||
Title,
|
||||
Text,
|
||||
Group,
|
||||
Button,
|
||||
Stack,
|
||||
Paper,
|
||||
Alert,
|
||||
Loader,
|
||||
} from "@mantine/core";
|
||||
import { IconArrowLeft } from "@tabler/icons-react";
|
||||
import { getRun } from "../api";
|
||||
import { StatusBadge } from "../components/StatusBadge";
|
||||
import { StepTimeline } from "../components/StepTimeline";
|
||||
import type { RunDetail as RunDetailType } from "../types";
|
||||
|
||||
export function RunDetail() {
|
||||
const { id } = useParams<{ id: string }>();
|
||||
const navigate = useNavigate();
|
||||
const [data, setData] = useState<RunDetailType | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
if (!id) return;
|
||||
try {
|
||||
setData(await getRun(id));
|
||||
setError(null);
|
||||
} catch (e) {
|
||||
setError((e as Error).message);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
// Auto-refresh while running.
|
||||
const interval = setInterval(() => {
|
||||
if (data?.run.Status === "running") {
|
||||
load();
|
||||
}
|
||||
}, 2000);
|
||||
return () => clearInterval(interval);
|
||||
}, [id, data?.run.Status]);
|
||||
|
||||
if (loading) return <Loader />;
|
||||
if (error) return <Alert color="red">{error}</Alert>;
|
||||
if (!data) return <Text>Not found</Text>;
|
||||
|
||||
const { run, steps } = data;
|
||||
const duration = run.FinishedAt
|
||||
? `${Math.round((new Date(run.FinishedAt).getTime() - new Date(run.StartedAt).getTime()) / 1000)}s`
|
||||
: "running...";
|
||||
|
||||
return (
|
||||
<Stack gap="md">
|
||||
<Group>
|
||||
<Button
|
||||
variant="subtle"
|
||||
size="xs"
|
||||
leftSection={<IconArrowLeft size={14} />}
|
||||
onClick={() => navigate(`/dags/${run.DagName}`)}
|
||||
>
|
||||
Back to {run.DagName}
|
||||
</Button>
|
||||
</Group>
|
||||
|
||||
<Group justify="space-between">
|
||||
<div>
|
||||
<Title order={2}>Run {run.ID.substring(0, 16)}...</Title>
|
||||
<Text size="sm" c="dimmed">
|
||||
{run.DagName} · {run.Trigger} ·{" "}
|
||||
{new Date(run.StartedAt).toLocaleString()}
|
||||
</Text>
|
||||
</div>
|
||||
<Group gap="xs">
|
||||
<StatusBadge status={run.Status} />
|
||||
<Text size="sm">{duration}</Text>
|
||||
</Group>
|
||||
</Group>
|
||||
|
||||
{run.Error && (
|
||||
<Alert color="red" title="Error">
|
||||
{run.Error}
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
<Paper p="md" withBorder>
|
||||
<Title order={4} mb="md">
|
||||
Steps ({steps?.length || 0})
|
||||
</Title>
|
||||
{steps?.length ? (
|
||||
<StepTimeline steps={steps} />
|
||||
) : (
|
||||
<Text size="sm" c="dimmed">
|
||||
No steps recorded
|
||||
</Text>
|
||||
)}
|
||||
</Paper>
|
||||
</Stack>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
export interface DagSummary {
|
||||
name: string;
|
||||
description?: string;
|
||||
schedule?: string[];
|
||||
tags?: string[];
|
||||
type?: string;
|
||||
file_path: string;
|
||||
valid: boolean;
|
||||
last_run?: DagRun;
|
||||
}
|
||||
|
||||
export interface DagRun {
|
||||
ID: string;
|
||||
DagName: string;
|
||||
DagPath: string;
|
||||
Status: string;
|
||||
Trigger: string;
|
||||
StartedAt: string;
|
||||
FinishedAt?: string;
|
||||
Error: string;
|
||||
}
|
||||
|
||||
export interface DagStepResult {
|
||||
ID: string;
|
||||
RunID: string;
|
||||
StepName: string;
|
||||
Status: string;
|
||||
ExitCode: number;
|
||||
Stdout: string;
|
||||
Stderr: string;
|
||||
StartedAt?: string;
|
||||
FinishedAt?: string;
|
||||
DurationMs: number;
|
||||
Error: string;
|
||||
}
|
||||
|
||||
export interface DagDetail {
|
||||
info: DagSummary;
|
||||
dag: {
|
||||
Name: string;
|
||||
Description: string;
|
||||
Type: string;
|
||||
Schedule: string[];
|
||||
Steps: { Name: string; ID: string; Command: string; Script: string; Depends: string[] }[];
|
||||
Env: Record<string, string>;
|
||||
Tags: string[];
|
||||
HandlerOn: { Failure: unknown[]; Success: unknown[] };
|
||||
};
|
||||
validation: {
|
||||
Valid: boolean;
|
||||
Errors: string[];
|
||||
Warnings: string[];
|
||||
Levels: string[][];
|
||||
};
|
||||
runs: DagRun[];
|
||||
}
|
||||
|
||||
export interface RunDetail {
|
||||
run: DagRun;
|
||||
steps: DagStepResult[];
|
||||
}
|
||||
|
||||
export interface SchedulerStatus {
|
||||
running: boolean;
|
||||
dags: { name: string; path: string; schedule: string; next_run: string }[];
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"useDefineForClassFields": true,
|
||||
"lib": ["ES2020", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"skipLibCheck": true,
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"isolatedModules": true,
|
||||
"moduleDetection": "force",
|
||||
"noEmit": true,
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"noUnusedLocals": true,
|
||||
"noUnusedParameters": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedSideEffectImports": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { defineConfig } from "vite";
|
||||
import react from "@vitejs/plugin-react";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [react()],
|
||||
server: {
|
||||
port: 5175,
|
||||
proxy: {
|
||||
"/api": "http://localhost:8090",
|
||||
},
|
||||
},
|
||||
build: {
|
||||
outDir: "dist",
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
module dag-engine
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
fn-registry v0.0.0-00010101000000-000000000000
|
||||
github.com/mattn/go-sqlite3 v1.14.37
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/ch-go v0.71.0 // indirect
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.44.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/apache/arrow-go/v18 v18.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/go-faster/city v1.0.1 // indirect
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/flatbuffers v25.1.24+incompatible // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.9.1 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.3 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
|
||||
github.com/marcboeker/go-duckdb v1.8.5 // indirect
|
||||
github.com/paulmach/orb v0.12.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.25 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/segmentio/asm v1.2.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.41.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect
|
||||
golang.org/x/mod v0.27.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.29.0 // indirect
|
||||
golang.org/x/tools v0.36.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
replace fn-registry => /home/lucas/fn_registry
|
||||
@@ -0,0 +1,168 @@
|
||||
github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM=
|
||||
github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.44.0 h1:9pxs5pRwIvhni5BDRPn/n5A8DeUod5TnBaeulFBX8EQ=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.44.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y=
|
||||
github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0=
|
||||
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
|
||||
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
|
||||
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
|
||||
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
|
||||
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o=
|
||||
github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc=
|
||||
github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
|
||||
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
|
||||
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0=
|
||||
github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8=
|
||||
github.com/mattn/go-sqlite3 v1.14.37 h1:3DOZp4cXis1cUIpCfXLtmlGolNLp2VEqhiB/PARNBIg=
|
||||
github.com/mattn/go-sqlite3 v1.14.37/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s=
|
||||
github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
|
||||
github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
|
||||
github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
|
||||
github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
|
||||
github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc=
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
|
||||
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
|
||||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
|
||||
gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func handleListDags(executor *Executor) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
dags, err := executor.ListDAGs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, dags)
|
||||
}
|
||||
}
|
||||
|
||||
func handleGetDag(executor *Executor) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
info, dag, validation, err := executor.GetDAG(name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Get recent runs.
|
||||
runs, _, _ := executor.store.ListRuns(dag.Name, 10, 0)
|
||||
|
||||
resp := map[string]interface{}{
|
||||
"info": info,
|
||||
"dag": dag,
|
||||
"validation": validation,
|
||||
"runs": runs,
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
}
|
||||
|
||||
func handleRunDag(executor *Executor) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
info, _, _, err := executor.GetDAG(name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Execute asynchronously.
|
||||
go func() {
|
||||
ctx := context.Background()
|
||||
executor.ExecuteDAG(ctx, info.FilePath, "api")
|
||||
}()
|
||||
|
||||
// Return run acknowledgment.
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{
|
||||
"status": "accepted",
|
||||
"dag": name,
|
||||
"message": "DAG execution started",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- JSON helpers ---
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, msg string) {
|
||||
writeJSON(w, status, map[string]string{"error": msg})
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func handleListRuns(executor *Executor) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
dagName := r.URL.Query().Get("dag")
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
offset, _ := strconv.Atoi(r.URL.Query().Get("offset"))
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
runs, total, err := executor.store.ListRuns(dagName, limit, offset)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"runs": runs,
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func handleGetRun(executor *Executor) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
run, err := executor.store.GetRun(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if run == nil {
|
||||
writeError(w, http.StatusNotFound, "run not found")
|
||||
return
|
||||
}
|
||||
|
||||
steps, err := executor.store.ListStepResults(id)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"run": run,
|
||||
"steps": steps,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import "net/http"
|
||||
|
||||
func handleSchedulerStart(scheduler *Scheduler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := scheduler.Start(); err != nil {
|
||||
writeError(w, http.StatusConflict, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "started"})
|
||||
}
|
||||
}
|
||||
|
||||
func handleSchedulerStop(scheduler *Scheduler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
scheduler.Stop()
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "stopped"})
|
||||
}
|
||||
}
|
||||
|
||||
func handleSchedulerStatus(scheduler *Scheduler) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
status := scheduler.Status()
|
||||
writeJSON(w, http.StatusOK, status)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"flag"
|
||||
"fmt"
|
||||
iofs "io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"fn-registry/functions/core"
|
||||
|
||||
"dag-engine/store"
|
||||
)
|
||||
|
||||
//go:embed all:frontend/dist
|
||||
var frontendDist embed.FS
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
printUsage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cmd := os.Args[1]
|
||||
args := os.Args[2:]
|
||||
|
||||
switch cmd {
|
||||
case "run":
|
||||
cmdRun(args)
|
||||
case "list":
|
||||
cmdList(args)
|
||||
case "status":
|
||||
cmdStatus(args)
|
||||
case "validate":
|
||||
cmdValidate(args)
|
||||
case "server":
|
||||
cmdServer(args)
|
||||
case "help", "-h", "--help":
|
||||
printUsage()
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown command: %s\n", cmd)
|
||||
printUsage()
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Println(`dag-engine — DAG workflow executor
|
||||
|
||||
Usage:
|
||||
dag-engine <command> [options]
|
||||
|
||||
Commands:
|
||||
run <path.yaml> Execute a DAG and show results
|
||||
list [dir] List DAGs with schedule and last status
|
||||
status [dag_name] Show execution history
|
||||
validate <path.yaml> Parse and validate without executing
|
||||
server Start HTTP server with web frontend
|
||||
|
||||
Server options:
|
||||
--port <port> HTTP port (default: 8090)
|
||||
--dags-dir <dir> DAGs directory (default: ~/dagu/dags)
|
||||
--db <path> SQLite database path (default: dag_engine.db)
|
||||
--scheduler Auto-start cron scheduler`)
|
||||
}
|
||||
|
||||
// --- CLI Commands ---
|
||||
|
||||
func cmdRun(args []string) {
|
||||
if len(args) < 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: dag-engine run <path.yaml>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
dagPath := args[0]
|
||||
cfg := DefaultConfig()
|
||||
|
||||
// Parse optional flags after the path.
|
||||
fs := flag.NewFlagSet("run", flag.ExitOnError)
|
||||
fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path")
|
||||
fs.Parse(args[1:])
|
||||
|
||||
db, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
executor := NewExecutor(db, filepath.Dir(dagPath))
|
||||
|
||||
fmt.Printf("Executing %s...\n", dagPath)
|
||||
ctx := context.Background()
|
||||
runID, err := executor.ExecuteDAG(ctx, dagPath, "manual")
|
||||
|
||||
// Print results.
|
||||
if runID != "" {
|
||||
run, _ := db.GetRun(runID)
|
||||
steps, _ := db.ListStepResults(runID)
|
||||
|
||||
if run != nil {
|
||||
fmt.Println()
|
||||
for _, s := range steps {
|
||||
icon := " "
|
||||
switch s.Status {
|
||||
case "success":
|
||||
icon = "OK"
|
||||
case "failed":
|
||||
icon = "!!"
|
||||
case "skipped":
|
||||
icon = "--"
|
||||
case "running":
|
||||
icon = ".."
|
||||
}
|
||||
fmt.Printf("[%s] %s (%dms)\n", icon, s.StepName, s.DurationMs)
|
||||
if s.Status == "failed" && s.Stderr != "" {
|
||||
for _, line := range strings.Split(strings.TrimSpace(s.Stderr), "\n") {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
dur := ""
|
||||
if run.FinishedAt != nil {
|
||||
dur = fmt.Sprintf(" (%s)", run.FinishedAt.Sub(run.StartedAt).Round(time.Millisecond))
|
||||
}
|
||||
fmt.Printf("Run %s: %s%s\n", runID, strings.ToUpper(run.Status), dur)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func cmdList(args []string) {
|
||||
cfg := DefaultConfig()
|
||||
if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
|
||||
cfg.DagsDir = args[0]
|
||||
args = args[1:]
|
||||
}
|
||||
|
||||
fs := flag.NewFlagSet("list", flag.ExitOnError)
|
||||
fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path")
|
||||
fs.Parse(args)
|
||||
|
||||
db, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
executor := NewExecutor(db, cfg.DagsDir)
|
||||
dags, err := executor.ListDAGs()
|
||||
if err != nil {
|
||||
log.Fatalf("list dags: %v", err)
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "NAME\tSCHEDULE\tTYPE\tTAGS\tLAST STATUS\tLAST RUN")
|
||||
for _, d := range dags {
|
||||
sched := strings.Join(d.Schedule, ", ")
|
||||
tags := strings.Join(d.Tags, ", ")
|
||||
lastStatus := "-"
|
||||
lastRun := "-"
|
||||
if d.LastRun != nil {
|
||||
lastStatus = d.LastRun.Status
|
||||
lastRun = d.LastRun.StartedAt.Format("2006-01-02 15:04")
|
||||
}
|
||||
typ := d.Type
|
||||
if typ == "" {
|
||||
typ = "chain"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", d.Name, sched, typ, tags, lastStatus, lastRun)
|
||||
}
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
func cmdStatus(args []string) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
fs := flag.NewFlagSet("status", flag.ExitOnError)
|
||||
fs.StringVar(&cfg.DBPath, "db", cfg.DBPath, "SQLite database path")
|
||||
limit := fs.Int("limit", 10, "number of runs to show")
|
||||
fs.Parse(args)
|
||||
|
||||
dagName := ""
|
||||
if fs.NArg() > 0 {
|
||||
dagName = fs.Arg(0)
|
||||
}
|
||||
|
||||
db, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
runs, total, err := db.ListRuns(dagName, *limit, 0)
|
||||
if err != nil {
|
||||
log.Fatalf("list runs: %v", err)
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintf(w, "Showing %d of %d runs", len(runs), total)
|
||||
if dagName != "" {
|
||||
fmt.Fprintf(w, " for %s", dagName)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, "RUN_ID\tDAG\tSTATUS\tTRIGGER\tSTARTED\tDURATION")
|
||||
for _, r := range runs {
|
||||
dur := "-"
|
||||
if r.FinishedAt != nil {
|
||||
dur = r.FinishedAt.Sub(r.StartedAt).Round(time.Millisecond).String()
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.DagName, r.Status, r.Trigger,
|
||||
r.StartedAt.Format("2006-01-02 15:04:05"), dur)
|
||||
}
|
||||
w.Flush()
|
||||
}
|
||||
|
||||
func cmdValidate(args []string) {
|
||||
if len(args) < 1 {
|
||||
fmt.Fprintln(os.Stderr, "usage: dag-engine validate <path.yaml>")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(args[0])
|
||||
if err != nil {
|
||||
log.Fatalf("read: %v", err)
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
log.Fatalf("parse error: %v", err)
|
||||
}
|
||||
|
||||
result := core.DagValidate(dag)
|
||||
|
||||
fmt.Printf("DAG: %s\n", dag.Name)
|
||||
fmt.Printf("Steps: %d\n", len(dag.Steps))
|
||||
fmt.Printf("Schedule: %v\n", dag.Schedule)
|
||||
fmt.Printf("Type: %s\n", dag.Type)
|
||||
|
||||
if result.Valid {
|
||||
fmt.Println("Validation: PASS")
|
||||
for i, level := range result.Levels {
|
||||
fmt.Printf(" Level %d: %v\n", i, level)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Validation: FAIL")
|
||||
for _, e := range result.Errors {
|
||||
fmt.Printf(" ERROR: %s\n", e)
|
||||
}
|
||||
}
|
||||
for _, w := range result.Warnings {
|
||||
fmt.Printf(" WARNING: %s\n", w)
|
||||
}
|
||||
|
||||
if !result.Valid {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Server Command ---
|
||||
|
||||
func cmdServer(args []string) {
|
||||
cfg := DefaultConfig()
|
||||
fs := flag.NewFlagSet("server", flag.ExitOnError)
|
||||
cfg.ParseFlags(fs, args)
|
||||
|
||||
db, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
log.Fatalf("open db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
executor := NewExecutor(db, cfg.DagsDir)
|
||||
scheduler := NewScheduler(executor, cfg.DagsDir)
|
||||
|
||||
// Prepare frontend FS.
|
||||
var feFS iofs.FS
|
||||
distFS, err := iofs.Sub(frontendDist, "frontend/dist")
|
||||
if err == nil {
|
||||
// Check if dist has content (built frontend exists).
|
||||
entries, _ := iofs.ReadDir(distFS, ".")
|
||||
if len(entries) > 0 {
|
||||
feFS = distFS
|
||||
log.Printf("serving frontend from embedded dist/")
|
||||
}
|
||||
}
|
||||
if feFS == nil {
|
||||
log.Printf("no frontend build found, API-only mode")
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
RegisterAPI(mux, executor, scheduler, feFS)
|
||||
|
||||
handler := corsMiddleware(loggingMiddleware(mux))
|
||||
|
||||
if cfg.AutoScheduler {
|
||||
if err := scheduler.Start(); err != nil {
|
||||
log.Printf("scheduler start: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf(":%d", cfg.Port)
|
||||
log.Printf("dag-engine server starting on http://0.0.0.0%s", addr)
|
||||
log.Printf("dags dir: %s", cfg.DagsDir)
|
||||
log.Printf("database: %s", cfg.DBPath)
|
||||
|
||||
srv := &http.Server{Addr: addr, Handler: handler}
|
||||
|
||||
// Graceful shutdown.
|
||||
go func() {
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-sigCh
|
||||
log.Println("shutting down...")
|
||||
scheduler.Stop()
|
||||
srv.Shutdown(context.Background())
|
||||
}()
|
||||
|
||||
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
|
||||
log.Fatalf("server: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// corsMiddleware adds permissive CORS headers for development.
|
||||
func corsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// loggingMiddleware logs each HTTP request with method, path and duration.
|
||||
func loggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
next.ServeHTTP(w, r)
|
||||
log.Printf("%s %s %s", r.Method, r.URL.Path, time.Since(start).Round(time.Millisecond))
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"fn-registry/functions/core"
|
||||
"fn-registry/functions/infra"
|
||||
)
|
||||
|
||||
// ScheduledDAG represents a DAG with a parsed cron schedule.
|
||||
type ScheduledDAG struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Schedule string `json:"schedule"`
|
||||
NextRun time.Time `json:"next_run"`
|
||||
}
|
||||
|
||||
// Scheduler manages cron-triggered DAG execution.
|
||||
type Scheduler struct {
|
||||
mu sync.Mutex
|
||||
running bool
|
||||
cancel context.CancelFunc
|
||||
dagsDir string
|
||||
executor *Executor
|
||||
dags []ScheduledDAG
|
||||
}
|
||||
|
||||
// NewScheduler creates a new scheduler.
|
||||
func NewScheduler(executor *Executor, dagsDir string) *Scheduler {
|
||||
return &Scheduler{
|
||||
executor: executor,
|
||||
dagsDir: dagsDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Start scans for DAGs with schedules and starts cron tickers for each.
|
||||
func (s *Scheduler) Start() error {
|
||||
s.mu.Lock()
|
||||
if s.running {
|
||||
s.mu.Unlock()
|
||||
return fmt.Errorf("scheduler already running")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s.cancel = cancel
|
||||
s.running = true
|
||||
s.mu.Unlock()
|
||||
|
||||
scheduled, err := s.scanDAGs()
|
||||
if err != nil {
|
||||
s.mu.Lock()
|
||||
s.running = false
|
||||
s.mu.Unlock()
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.dags = scheduled
|
||||
s.mu.Unlock()
|
||||
|
||||
log.Printf("[scheduler] started with %d DAGs", len(scheduled))
|
||||
|
||||
for _, dag := range scheduled {
|
||||
dag := dag
|
||||
go s.runTicker(ctx, dag)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop cancels all tickers and stops the scheduler.
|
||||
func (s *Scheduler) Stop() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if !s.running {
|
||||
return
|
||||
}
|
||||
s.cancel()
|
||||
s.running = false
|
||||
s.dags = nil
|
||||
log.Printf("[scheduler] stopped")
|
||||
}
|
||||
|
||||
// IsRunning returns true if the scheduler is active.
|
||||
func (s *Scheduler) IsRunning() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.running
|
||||
}
|
||||
|
||||
// Status returns the list of scheduled DAGs with their next run time.
|
||||
type SchedulerStatus struct {
|
||||
Running bool `json:"running"`
|
||||
DAGs []ScheduledDAG `json:"dags"`
|
||||
}
|
||||
|
||||
func (s *Scheduler) Status() SchedulerStatus {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return SchedulerStatus{
|
||||
Running: s.running,
|
||||
DAGs: s.dags,
|
||||
}
|
||||
}
|
||||
|
||||
// scanDAGs reads the dags directory and returns DAGs that have cron schedules.
|
||||
func (s *Scheduler) scanDAGs() ([]ScheduledDAG, error) {
|
||||
entries, err := os.ReadDir(s.dagsDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var scheduled []ScheduledDAG
|
||||
for _, entry := range entries {
|
||||
ext := filepath.Ext(entry.Name())
|
||||
if ext != ".yaml" && ext != ".yml" {
|
||||
continue
|
||||
}
|
||||
|
||||
path := filepath.Join(s.dagsDir, entry.Name())
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
dag, err := core.DagParse(data)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, expr := range dag.Schedule {
|
||||
sched, err := core.ParseCronExpr(strings.TrimSpace(expr))
|
||||
if err != nil {
|
||||
log.Printf("[scheduler] invalid cron %q in %s: %v", expr, dag.Name, err)
|
||||
continue
|
||||
}
|
||||
next := core.NextCronTime(sched, time.Now())
|
||||
scheduled = append(scheduled, ScheduledDAG{
|
||||
Name: dag.Name,
|
||||
Path: path,
|
||||
Schedule: expr,
|
||||
NextRun: next,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return scheduled, nil
|
||||
}
|
||||
|
||||
// runTicker starts a cron ticker for a single DAG schedule.
|
||||
func (s *Scheduler) runTicker(ctx context.Context, dag ScheduledDAG) {
|
||||
sched, err := core.ParseCronExpr(strings.TrimSpace(dag.Schedule))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert core.CronSchedule to infra.CronTickerSchedule.
|
||||
tickerSched := infra.CronTickerSchedule{
|
||||
Minute: sched.Minute,
|
||||
Hour: sched.Hour,
|
||||
DayOfMonth: sched.DayOfMonth,
|
||||
Month: sched.Month,
|
||||
DayOfWeek: sched.DayOfWeek,
|
||||
}
|
||||
|
||||
ch := infra.CronTicker(tickerSched, ctx)
|
||||
log.Printf("[scheduler] ticker started for %s (%s), next: %s", dag.Name, dag.Schedule, dag.NextRun.Format(time.RFC3339))
|
||||
|
||||
for t := range ch {
|
||||
log.Printf("[scheduler] triggered %s at %s", dag.Name, t.Format(time.RFC3339))
|
||||
go func() {
|
||||
runID, err := s.executor.ExecuteDAG(ctx, dag.Path, "cron")
|
||||
if err != nil {
|
||||
log.Printf("[scheduler] %s failed: %v (run: %s)", dag.Name, err, runID)
|
||||
} else {
|
||||
log.Printf("[scheduler] %s completed (run: %s)", dag.Name, runID)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
CREATE TABLE IF NOT EXISTS dag_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
dag_name TEXT NOT NULL,
|
||||
dag_path TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','success','failed','cancelled')),
|
||||
trigger TEXT NOT NULL DEFAULT 'manual' CHECK(trigger IN ('manual','cron','api')),
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
error TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dag_step_results (
|
||||
id TEXT PRIMARY KEY,
|
||||
run_id TEXT NOT NULL REFERENCES dag_runs(id) ON DELETE CASCADE,
|
||||
step_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','running','success','failed','skipped')),
|
||||
exit_code INTEGER NOT NULL DEFAULT -1,
|
||||
stdout TEXT NOT NULL DEFAULT '',
|
||||
stderr TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
duration_ms INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_dag_name ON dag_runs(dag_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_status ON dag_runs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_started ON dag_runs(started_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_step_results_run ON dag_step_results(run_id);
|
||||
@@ -0,0 +1,231 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
//go:embed migrations/001_init.sql
|
||||
var migrationSQL string
|
||||
|
||||
// DB wraps a SQLite connection for DAG run persistence.
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
path string
|
||||
}
|
||||
|
||||
// Open opens or creates a DAG engine database at the given path.
|
||||
func Open(path string) (*DB, error) {
|
||||
conn, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000&_foreign_keys=on")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("store: open %s: %w", path, err)
|
||||
}
|
||||
if _, err := conn.Exec(migrationSQL); err != nil {
|
||||
conn.Close()
|
||||
return nil, fmt.Errorf("store: migrate: %w", err)
|
||||
}
|
||||
return &DB{conn: conn, path: path}, nil
|
||||
}
|
||||
|
||||
// Close closes the database connection.
|
||||
func (db *DB) Close() error {
|
||||
return db.conn.Close()
|
||||
}
|
||||
|
||||
// --- DagRun CRUD ---
|
||||
|
||||
// DagRun mirrors infra.DagRun for the store layer.
|
||||
type DagRun struct {
|
||||
ID string
|
||||
DagName string
|
||||
DagPath string
|
||||
Status string
|
||||
Trigger string
|
||||
StartedAt time.Time
|
||||
FinishedAt *time.Time
|
||||
Error string
|
||||
}
|
||||
|
||||
// CreateRun inserts a new run record.
|
||||
func (db *DB) CreateRun(run *DagRun) error {
|
||||
_, err := db.conn.Exec(
|
||||
`INSERT INTO dag_runs (id, dag_name, dag_path, status, trigger, started_at, error)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
run.ID, run.DagName, run.DagPath, run.Status, run.Trigger,
|
||||
run.StartedAt.Format(time.RFC3339), run.Error,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateRunStatus updates a run's status and optionally its finished_at and error.
|
||||
func (db *DB) UpdateRunStatus(id, status string, finishedAt *time.Time, errMsg string) error {
|
||||
var fin *string
|
||||
if finishedAt != nil {
|
||||
s := finishedAt.Format(time.RFC3339)
|
||||
fin = &s
|
||||
}
|
||||
_, err := db.conn.Exec(
|
||||
`UPDATE dag_runs SET status=?, finished_at=?, error=? WHERE id=?`,
|
||||
status, fin, errMsg, id,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetRun retrieves a single run by ID.
|
||||
func (db *DB) GetRun(id string) (*DagRun, error) {
|
||||
row := db.conn.QueryRow(
|
||||
`SELECT id, dag_name, dag_path, status, trigger, started_at, finished_at, error
|
||||
FROM dag_runs WHERE id=?`, id,
|
||||
)
|
||||
return scanRun(row)
|
||||
}
|
||||
|
||||
// ListRuns returns runs, newest first, with optional dag name filter.
|
||||
func (db *DB) ListRuns(dagName string, limit, offset int) ([]DagRun, int, error) {
|
||||
var total int
|
||||
var args []interface{}
|
||||
where := ""
|
||||
if dagName != "" {
|
||||
where = " WHERE dag_name=?"
|
||||
args = append(args, dagName)
|
||||
}
|
||||
err := db.conn.QueryRow("SELECT COUNT(*) FROM dag_runs"+where, args...).Scan(&total)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
query := "SELECT id, dag_name, dag_path, status, trigger, started_at, finished_at, error FROM dag_runs" +
|
||||
where + " ORDER BY started_at DESC LIMIT ? OFFSET ?"
|
||||
args = append(args, limit, offset)
|
||||
|
||||
rows, err := db.conn.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var runs []DagRun
|
||||
for rows.Next() {
|
||||
r, err := scanRunRows(rows)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
runs = append(runs, *r)
|
||||
}
|
||||
return runs, total, rows.Err()
|
||||
}
|
||||
|
||||
// --- DagStepResult CRUD ---
|
||||
|
||||
// DagStepResult mirrors infra.DagStepResult for the store layer.
|
||||
type DagStepResult struct {
|
||||
ID string
|
||||
RunID string
|
||||
StepName string
|
||||
Status string
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
StartedAt *time.Time
|
||||
FinishedAt *time.Time
|
||||
DurationMs int64
|
||||
Error string
|
||||
}
|
||||
|
||||
// InsertStepResult inserts a new step result.
|
||||
func (db *DB) InsertStepResult(r *DagStepResult) error {
|
||||
var startedAt, finishedAt *string
|
||||
if r.StartedAt != nil {
|
||||
s := r.StartedAt.Format(time.RFC3339)
|
||||
startedAt = &s
|
||||
}
|
||||
if r.FinishedAt != nil {
|
||||
s := r.FinishedAt.Format(time.RFC3339)
|
||||
finishedAt = &s
|
||||
}
|
||||
_, err := db.conn.Exec(
|
||||
`INSERT INTO dag_step_results (id, run_id, step_name, status, exit_code, stdout, stderr, started_at, finished_at, duration_ms, error)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
r.ID, r.RunID, r.StepName, r.Status, r.ExitCode, r.Stdout, r.Stderr,
|
||||
startedAt, finishedAt, r.DurationMs, r.Error,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateStepResult updates a step result by ID.
|
||||
func (db *DB) UpdateStepResult(id, status string, exitCode int, stdout, stderr string, finishedAt *time.Time, durationMs int64, errMsg string) error {
|
||||
var fin *string
|
||||
if finishedAt != nil {
|
||||
s := finishedAt.Format(time.RFC3339)
|
||||
fin = &s
|
||||
}
|
||||
_, err := db.conn.Exec(
|
||||
`UPDATE dag_step_results SET status=?, exit_code=?, stdout=?, stderr=?, finished_at=?, duration_ms=?, error=? WHERE id=?`,
|
||||
status, exitCode, stdout, stderr, fin, durationMs, errMsg, id,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListStepResults returns all step results for a given run.
|
||||
func (db *DB) ListStepResults(runID string) ([]DagStepResult, error) {
|
||||
rows, err := db.conn.Query(
|
||||
`SELECT id, run_id, step_name, status, exit_code, stdout, stderr, started_at, finished_at, duration_ms, error
|
||||
FROM dag_step_results WHERE run_id=? ORDER BY started_at ASC`, runID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []DagStepResult
|
||||
for rows.Next() {
|
||||
var r DagStepResult
|
||||
var startedAt, finishedAt sql.NullString
|
||||
if err := rows.Scan(&r.ID, &r.RunID, &r.StepName, &r.Status, &r.ExitCode,
|
||||
&r.Stdout, &r.Stderr, &startedAt, &finishedAt, &r.DurationMs, &r.Error); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if startedAt.Valid {
|
||||
t, _ := time.Parse(time.RFC3339, startedAt.String)
|
||||
r.StartedAt = &t
|
||||
}
|
||||
if finishedAt.Valid {
|
||||
t, _ := time.Parse(time.RFC3339, finishedAt.String)
|
||||
r.FinishedAt = &t
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
return results, rows.Err()
|
||||
}
|
||||
|
||||
// --- scan helpers ---
|
||||
|
||||
type scanner interface {
|
||||
Scan(dest ...interface{}) error
|
||||
}
|
||||
|
||||
func scanRun(s scanner) (*DagRun, error) {
|
||||
var r DagRun
|
||||
var startedAt string
|
||||
var finishedAt sql.NullString
|
||||
if err := s.Scan(&r.ID, &r.DagName, &r.DagPath, &r.Status, &r.Trigger, &startedAt, &finishedAt, &r.Error); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
r.StartedAt, _ = time.Parse(time.RFC3339, startedAt)
|
||||
if finishedAt.Valid {
|
||||
t, _ := time.Parse(time.RFC3339, finishedAt.String)
|
||||
r.FinishedAt = &t
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func scanRunRows(rows *sql.Rows) (*DagRun, error) {
|
||||
return scanRun(rows)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"dag-engine": {
|
||||
"enabled": false,
|
||||
"enabled": true,
|
||||
"issue": "0007",
|
||||
"description": "Sistema propio de orquestacion de DAGs para reemplazar Dagu. Incluye parser YAML, executor con paralelismo, process manager, execution store y scheduler cron."
|
||||
"description": "Sistema propio de orquestacion de DAGs para reemplazar Dagu. Incluye parser YAML, executor con paralelismo, process manager, execution store SQLite, scheduler cron, CLI y web frontend."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,9 +8,9 @@
|
||||
| 0004 | Jupyter discover multiple instances | completado | — | feature | — |
|
||||
| 0005 | Jupyter write batch | completado | — | feature | — |
|
||||
| 0006 | Jupyter exec outputs keyerror | completado | — | bugfix | — |
|
||||
| **0007a** | **DAG engine: core (parse, validate, topo sort)** | pendiente | alta | feature | 0007b-e |
|
||||
| **0007b** | **DAG engine: process manager (spawn, wait, kill)** | pendiente | alta | feature | 0007e |
|
||||
| **0007c** | **DAG engine: execution store (SQLite)** | pendiente | alta | feature | 0007e |
|
||||
| **0007d** | **DAG engine: scheduler (cron parser, ticker)** | pendiente | media | feature | 0007e |
|
||||
| **0007e** | **DAG engine: app CLI que reemplaza Dagu** | pendiente | alta | feature | — |
|
||||
| [0007a](completed/0007a-dag-core.md) | DAG engine: core (parse, validate, topo sort) | completado | alta | feature | 0007b-e |
|
||||
| [0007b](completed/0007b-process-manager.md) | DAG engine: process manager (spawn, wait, kill) | completado | alta | feature | 0007e |
|
||||
| [0007c](completed/0007c-execution-store.md) | DAG engine: execution store (SQLite) | completado | alta | feature | 0007e |
|
||||
| [0007d](completed/0007d-scheduler.md) | DAG engine: scheduler (cron match) | completado | media | feature | 0007e |
|
||||
| [0007e](completed/0007e-dag-executor-app.md) | DAG engine: CLI + web app que reemplaza Dagu | completado | alta | feature | — |
|
||||
| **0008** | **SQLite API Web** | pendiente | alta | feature | — |
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package core
|
||||
|
||||
import "time"
|
||||
|
||||
// CronMatch returns true if time t matches all fields of the cron schedule.
|
||||
func CronMatch(sched CronSchedule, t time.Time) bool {
|
||||
return intIn(t.Minute(), sched.Minute) &&
|
||||
intIn(t.Hour(), sched.Hour) &&
|
||||
intIn(t.Day(), sched.DayOfMonth) &&
|
||||
intIn(int(t.Month()), sched.Month) &&
|
||||
intIn(int(t.Weekday()), sched.DayOfWeek)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
name: cron_match
|
||||
kind: function
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
purity: pure
|
||||
signature: "func CronMatch(sched CronSchedule, t time.Time) bool"
|
||||
description: "Verifica si un instante de tiempo coincide con un cron schedule. Compara los 5 campos (minuto, hora, dia del mes, mes, dia de la semana) y retorna true si todos coinciden."
|
||||
tags: [cron, scheduling, matching, time, pure]
|
||||
uses_functions: []
|
||||
uses_types: [cron_schedule_go_core]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: ""
|
||||
imports: [time]
|
||||
params:
|
||||
- name: sched
|
||||
desc: "CronSchedule con listas de valores validos por campo (resultado de ParseCronExpr)"
|
||||
- name: t
|
||||
desc: "instante de tiempo a verificar contra el schedule"
|
||||
output: "true si t coincide con todos los campos del cron schedule"
|
||||
tested: true
|
||||
tests:
|
||||
- "9:00 AM coincide con 0 9 * * *"
|
||||
- "9:15 AM NO coincide con 0 9 * * *"
|
||||
- "lunes a las 9 coincide con 0 9 * * 1"
|
||||
- "domingo a las 9 NO coincide con 0 9 * * 1"
|
||||
- "wildcard * coincide con cualquier valor"
|
||||
- "specific month"
|
||||
test_file_path: "functions/core/cron_match_test.go"
|
||||
file_path: "functions/core/cron_match.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
sched, _ := ParseCronExpr("0 9 * * *")
|
||||
t := time.Date(2026, 4, 11, 9, 0, 0, 0, time.UTC)
|
||||
CronMatch(sched, t) // true
|
||||
|
||||
t2 := time.Date(2026, 4, 11, 10, 0, 0, 0, time.UTC)
|
||||
CronMatch(sched, t2) // false
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion pura. Usa AND semantics para day_of_month y day_of_week (ambos deben coincidir), igual que NextCronTime en el mismo paquete.
|
||||
Reutiliza el helper intIn definido en next_cron_time.go (mismo paquete core).
|
||||
@@ -0,0 +1,88 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCronMatch(t *testing.T) {
|
||||
t.Run("9:00 AM coincide con 0 9 * * *", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("0 9 * * *")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
ts := time.Date(2026, 4, 11, 9, 0, 0, 0, time.UTC)
|
||||
if !CronMatch(sched, ts) {
|
||||
t.Errorf("expected match for 9:00 AM with '0 9 * * *'")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("9:15 AM NO coincide con 0 9 * * *", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("0 9 * * *")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
ts := time.Date(2026, 4, 11, 9, 15, 0, 0, time.UTC)
|
||||
if CronMatch(sched, ts) {
|
||||
t.Errorf("expected no match for 9:15 AM with '0 9 * * *'")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("lunes a las 9 coincide con 0 9 * * 1", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("0 9 * * 1")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
// 2026-04-13 is a Monday
|
||||
ts := time.Date(2026, 4, 13, 9, 0, 0, 0, time.UTC)
|
||||
if !CronMatch(sched, ts) {
|
||||
t.Errorf("expected match for Monday 9:00 with '0 9 * * 1'")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("domingo a las 9 NO coincide con 0 9 * * 1", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("0 9 * * 1")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
// 2026-04-12 is a Sunday (weekday 0)
|
||||
ts := time.Date(2026, 4, 12, 9, 0, 0, 0, time.UTC)
|
||||
if CronMatch(sched, ts) {
|
||||
t.Errorf("expected no match for Sunday 9:00 with '0 9 * * 1'")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wildcard * coincide con cualquier valor", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("*/15 * * * *")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
// 9:30 should match */15 (0,15,30,45)
|
||||
ts := time.Date(2026, 4, 11, 9, 30, 0, 0, time.UTC)
|
||||
if !CronMatch(sched, ts) {
|
||||
t.Errorf("expected match for 9:30 with '*/15 * * * *'")
|
||||
}
|
||||
// 9:07 should NOT match */15
|
||||
ts2 := time.Date(2026, 4, 11, 9, 7, 0, 0, time.UTC)
|
||||
if CronMatch(sched, ts2) {
|
||||
t.Errorf("expected no match for 9:07 with '*/15 * * * *'")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("specific month", func(t *testing.T) {
|
||||
sched, err := ParseCronExpr("0 0 1 4 *")
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCronExpr: %v", err)
|
||||
}
|
||||
// April 1st matches
|
||||
ts := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||
if !CronMatch(sched, ts) {
|
||||
t.Errorf("expected match for April 1 with '0 0 1 4 *'")
|
||||
}
|
||||
// March 1st does NOT match
|
||||
ts2 := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
|
||||
if CronMatch(sched, ts2) {
|
||||
t.Errorf("expected no match for March 1 with '0 0 1 4 *'")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package core
|
||||
|
||||
// DagContinueOn controls whether a step continues on failure/skip.
|
||||
type DagContinueOn struct {
|
||||
Failure bool
|
||||
Skipped bool
|
||||
}
|
||||
|
||||
// DagRetryPolicy configures automatic retries for a step.
|
||||
type DagRetryPolicy struct {
|
||||
Limit int
|
||||
IntervalSec int
|
||||
}
|
||||
|
||||
// DagStep represents a single step in a DAG workflow.
|
||||
type DagStep struct {
|
||||
Name string
|
||||
ID string
|
||||
Description string
|
||||
Command string
|
||||
Script string
|
||||
Args []string
|
||||
Shell string
|
||||
Dir string
|
||||
Depends []string
|
||||
Env map[string]string
|
||||
ContinueOn DagContinueOn
|
||||
RetryPolicy DagRetryPolicy
|
||||
TimeoutSec int
|
||||
Output string
|
||||
Tags []string
|
||||
}
|
||||
|
||||
// DagHandlers contains lifecycle handler steps.
|
||||
type DagHandlers struct {
|
||||
Init []DagStep
|
||||
Success []DagStep
|
||||
Failure []DagStep
|
||||
Exit []DagStep
|
||||
}
|
||||
|
||||
// DagDefinition is a complete DAG workflow parsed from YAML.
|
||||
type DagDefinition struct {
|
||||
Name string
|
||||
Description string
|
||||
Group string
|
||||
Type string // "graph" or "" (chain/sequential)
|
||||
WorkingDir string
|
||||
Shell string
|
||||
Env map[string]string
|
||||
Schedule []string
|
||||
Steps []DagStep
|
||||
HandlerOn DagHandlers
|
||||
Tags []string
|
||||
TimeoutSec int
|
||||
FilePath string
|
||||
}
|
||||
|
||||
// DagValidationResult contains validation output.
|
||||
type DagValidationResult struct {
|
||||
Valid bool
|
||||
Errors []string
|
||||
Warnings []string
|
||||
Levels [][]string // topological levels (step names/IDs)
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// rawDagStep is the loosely-typed intermediate representation of a DAG step.
|
||||
type rawDagStep struct {
|
||||
Name string `yaml:"name"`
|
||||
ID string `yaml:"id"`
|
||||
Description string `yaml:"description"`
|
||||
Command string `yaml:"command"`
|
||||
Script string `yaml:"script"`
|
||||
Args []string `yaml:"args"`
|
||||
Shell string `yaml:"shell"`
|
||||
Dir string `yaml:"dir"`
|
||||
WorkingDir string `yaml:"working_dir"`
|
||||
Depends []string `yaml:"depends"`
|
||||
Env interface{} `yaml:"env"`
|
||||
ContinueOn rawDagContinueOn `yaml:"continue_on"`
|
||||
RetryPolicy rawDagRetryPolicy `yaml:"retry_policy"`
|
||||
TimeoutSec int `yaml:"timeout_sec"`
|
||||
Output string `yaml:"output"`
|
||||
Tags []string `yaml:"tags"`
|
||||
}
|
||||
|
||||
type rawDagContinueOn struct {
|
||||
Failure bool `yaml:"failure"`
|
||||
Skipped bool `yaml:"skipped"`
|
||||
}
|
||||
|
||||
type rawDagRetryPolicy struct {
|
||||
Limit int `yaml:"limit"`
|
||||
IntervalSec int `yaml:"interval_sec"`
|
||||
}
|
||||
|
||||
// rawDagHandlers handles both handler_on and handlers aliases.
|
||||
type rawDagHandlers struct {
|
||||
Init interface{} `yaml:"init"`
|
||||
Success interface{} `yaml:"success"`
|
||||
Failure interface{} `yaml:"failure"`
|
||||
Exit interface{} `yaml:"exit"`
|
||||
}
|
||||
|
||||
// rawDag is the loosely-typed intermediate representation of a DAG YAML.
|
||||
type rawDag struct {
|
||||
Name string `yaml:"name"`
|
||||
Description string `yaml:"description"`
|
||||
Group string `yaml:"group"`
|
||||
Type string `yaml:"type"`
|
||||
WorkingDir string `yaml:"working_dir"`
|
||||
Shell string `yaml:"shell"`
|
||||
Env interface{} `yaml:"env"`
|
||||
Schedule interface{} `yaml:"schedule"`
|
||||
Steps []rawDagStep `yaml:"steps"`
|
||||
HandlerOn *rawDagHandlers `yaml:"handler_on"`
|
||||
Handlers *rawDagHandlers `yaml:"handlers"`
|
||||
Tags []string `yaml:"tags"`
|
||||
TimeoutSec int `yaml:"timeout_sec"`
|
||||
}
|
||||
|
||||
// DagParse parses a YAML DAG definition compatible with Dagu format.
|
||||
// Handles schedule as string or list, env as list of single-key maps,
|
||||
// handler_on and handlers as aliases, and steps with command/script/depends.
|
||||
func DagParse(data []byte) (DagDefinition, error) {
|
||||
var raw rawDag
|
||||
if err := yaml.Unmarshal(data, &raw); err != nil {
|
||||
return DagDefinition{}, fmt.Errorf("dag_parse: yaml unmarshal: %w", err)
|
||||
}
|
||||
|
||||
def := DagDefinition{
|
||||
Name: raw.Name,
|
||||
Description: raw.Description,
|
||||
Group: raw.Group,
|
||||
Type: raw.Type,
|
||||
WorkingDir: raw.WorkingDir,
|
||||
Shell: raw.Shell,
|
||||
Tags: raw.Tags,
|
||||
TimeoutSec: raw.TimeoutSec,
|
||||
}
|
||||
|
||||
// Normalize env (list of single-key maps or plain map).
|
||||
dagEnv, err := normalizeEnv(raw.Env)
|
||||
if err != nil {
|
||||
return DagDefinition{}, fmt.Errorf("dag_parse: env: %w", err)
|
||||
}
|
||||
def.Env = dagEnv
|
||||
|
||||
// Normalize schedule (string or list).
|
||||
def.Schedule = normalizeSchedule(raw.Schedule)
|
||||
|
||||
// Normalize steps.
|
||||
steps := make([]DagStep, 0, len(raw.Steps))
|
||||
for i, rs := range raw.Steps {
|
||||
step, err := normalizeStep(rs)
|
||||
if err != nil {
|
||||
return DagDefinition{}, fmt.Errorf("dag_parse: step[%d]: %w", i, err)
|
||||
}
|
||||
steps = append(steps, step)
|
||||
}
|
||||
def.Steps = steps
|
||||
|
||||
// Normalize handlers: handler_on takes precedence, fallback to handlers.
|
||||
rawHandlers := raw.HandlerOn
|
||||
if rawHandlers == nil {
|
||||
rawHandlers = raw.Handlers
|
||||
}
|
||||
if rawHandlers != nil {
|
||||
handlers, err := normalizeHandlers(rawHandlers)
|
||||
if err != nil {
|
||||
return DagDefinition{}, fmt.Errorf("dag_parse: handlers: %w", err)
|
||||
}
|
||||
def.HandlerOn = handlers
|
||||
}
|
||||
|
||||
return def, nil
|
||||
}
|
||||
|
||||
// normalizeSchedule converts schedule from string or []interface{} to []string.
|
||||
func normalizeSchedule(v interface{}) []string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
s := strings.TrimSpace(t)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
return []string{s}
|
||||
case []interface{}:
|
||||
result := make([]string, 0, len(t))
|
||||
for _, item := range t {
|
||||
if s, ok := item.(string); ok {
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeEnv converts env from Dagu format (list of single-key maps) or plain map to map[string]string.
|
||||
func normalizeEnv(v interface{}) (map[string]string, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
switch t := v.(type) {
|
||||
case map[string]interface{}:
|
||||
// Plain YAML map.
|
||||
result := make(map[string]string, len(t))
|
||||
for k, val := range t {
|
||||
result[k] = fmt.Sprintf("%v", val)
|
||||
}
|
||||
return result, nil
|
||||
case []interface{}:
|
||||
// Dagu format: list of single-key maps [KEY: value, KEY2: value2].
|
||||
result := make(map[string]string)
|
||||
for _, item := range t {
|
||||
m, ok := item.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for k, val := range m {
|
||||
result[k] = fmt.Sprintf("%v", val)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected env type %T", v)
|
||||
}
|
||||
|
||||
// normalizeStep converts a rawDagStep to a DagStep.
|
||||
func normalizeStep(rs rawDagStep) (DagStep, error) {
|
||||
stepEnv, err := normalizeEnv(rs.Env)
|
||||
if err != nil {
|
||||
return DagStep{}, fmt.Errorf("env: %w", err)
|
||||
}
|
||||
|
||||
// working_dir is an alias for dir at the step level.
|
||||
dir := rs.Dir
|
||||
if dir == "" {
|
||||
dir = rs.WorkingDir
|
||||
}
|
||||
|
||||
return DagStep{
|
||||
Name: rs.Name,
|
||||
ID: rs.ID,
|
||||
Description: rs.Description,
|
||||
Command: rs.Command,
|
||||
Script: rs.Script,
|
||||
Args: rs.Args,
|
||||
Shell: rs.Shell,
|
||||
Dir: dir,
|
||||
Depends: rs.Depends,
|
||||
Env: stepEnv,
|
||||
ContinueOn: DagContinueOn{
|
||||
Failure: rs.ContinueOn.Failure,
|
||||
Skipped: rs.ContinueOn.Skipped,
|
||||
},
|
||||
RetryPolicy: DagRetryPolicy{
|
||||
Limit: rs.RetryPolicy.Limit,
|
||||
IntervalSec: rs.RetryPolicy.IntervalSec,
|
||||
},
|
||||
TimeoutSec: rs.TimeoutSec,
|
||||
Output: rs.Output,
|
||||
Tags: rs.Tags,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// normalizeHandlers converts rawDagHandlers to DagHandlers.
|
||||
// Each handler field can be a single step object or a list of steps.
|
||||
func normalizeHandlers(rh *rawDagHandlers) (DagHandlers, error) {
|
||||
var h DagHandlers
|
||||
var err error
|
||||
h.Init, err = normalizeHandlerField(rh.Init)
|
||||
if err != nil {
|
||||
return DagHandlers{}, fmt.Errorf("init: %w", err)
|
||||
}
|
||||
h.Success, err = normalizeHandlerField(rh.Success)
|
||||
if err != nil {
|
||||
return DagHandlers{}, fmt.Errorf("success: %w", err)
|
||||
}
|
||||
h.Failure, err = normalizeHandlerField(rh.Failure)
|
||||
if err != nil {
|
||||
return DagHandlers{}, fmt.Errorf("failure: %w", err)
|
||||
}
|
||||
h.Exit, err = normalizeHandlerField(rh.Exit)
|
||||
if err != nil {
|
||||
return DagHandlers{}, fmt.Errorf("exit: %w", err)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// normalizeHandlerField converts a handler field (single step or list) to []DagStep.
|
||||
func normalizeHandlerField(v interface{}) ([]DagStep, error) {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Re-marshal and unmarshal to handle polymorphic types cleanly.
|
||||
b, err := yaml.Marshal(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("re-marshal handler: %w", err)
|
||||
}
|
||||
|
||||
// Try as a list first.
|
||||
var rawList []rawDagStep
|
||||
if err := yaml.Unmarshal(b, &rawList); err == nil && len(rawList) > 0 {
|
||||
steps := make([]DagStep, 0, len(rawList))
|
||||
for i, rs := range rawList {
|
||||
step, err := normalizeStep(rs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("step[%d]: %w", i, err)
|
||||
}
|
||||
steps = append(steps, step)
|
||||
}
|
||||
return steps, nil
|
||||
}
|
||||
|
||||
// Try as single step.
|
||||
var rs rawDagStep
|
||||
if err := yaml.Unmarshal(b, &rs); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal handler step: %w", err)
|
||||
}
|
||||
step, err := normalizeStep(rs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// If the step has no meaningful content, return nil.
|
||||
if step.Name == "" && step.ID == "" && step.Command == "" && step.Script == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return []DagStep{step}, nil
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
name: dag_parse
|
||||
kind: function
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
purity: pure
|
||||
signature: "func DagParse(data []byte) (DagDefinition, error)"
|
||||
description: "Parsea YAML de definicion de DAG en formato compatible con Dagu. Soporta schedule como string o lista, env como lista de maps single-key (formato Dagu), handler_on y handlers como aliases, steps con command/script/depends/continue_on, y type graph."
|
||||
tags: [dag, yaml, parsing, workflow, dagu, pure]
|
||||
uses_functions: []
|
||||
uses_types: [dag_definition_go_core, dag_step_go_core, dag_handlers_go_core]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: ""
|
||||
imports: [fmt, strings, gopkg.in/yaml.v3]
|
||||
params:
|
||||
- name: data
|
||||
desc: "contenido YAML de un archivo de definicion de DAG en formato Dagu"
|
||||
output: "DagDefinition con todos los campos normalizados; error si el YAML es sintaticamente invalido"
|
||||
tested: true
|
||||
tests:
|
||||
- "parsea DAG simple con steps y depends"
|
||||
- "parsea schedule como string y como lista"
|
||||
- "parsea env en formato lista de maps"
|
||||
- "parsea handler_on y handlers como alias"
|
||||
- "parsea continue_on y working_dir a nivel step"
|
||||
- "parsea type graph"
|
||||
test_file_path: "functions/core/dag_parse_test.go"
|
||||
file_path: "functions/core/dag_parse.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
data := []byte(`
|
||||
name: mi-dag
|
||||
schedule: "0 9 * * *"
|
||||
steps:
|
||||
- name: hello
|
||||
command: echo "hello"
|
||||
- name: world
|
||||
command: echo "world"
|
||||
depends: [hello]
|
||||
`)
|
||||
dag, err := DagParse(data)
|
||||
// dag.Name = "mi-dag"
|
||||
// dag.Schedule = ["0 9 * * *"]
|
||||
// dag.Steps[1].Depends = ["hello"]
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion pura (el YAML es inmutable, no hay I/O). Internamente usa un struct rawDag para deserializar loosely y luego normaliza campos polimorficos. La estrategia de normalizacion: schedule string->[]string, env lista->map, handlers single-o-lista->[]DagStep. handler_on tiene precedencia sobre handlers si ambos estan presentes.
|
||||
@@ -0,0 +1,213 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDagParse(t *testing.T) {
|
||||
t.Run("parsea DAG simple con steps y depends", func(t *testing.T) {
|
||||
data := []byte(`
|
||||
name: example
|
||||
description: Example workflow
|
||||
steps:
|
||||
- name: hello
|
||||
command: echo "Hello!"
|
||||
- name: list_files
|
||||
command: ls -la /tmp
|
||||
depends:
|
||||
- hello
|
||||
`)
|
||||
dag, err := DagParse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if dag.Name != "example" {
|
||||
t.Errorf("Name: got %q, want %q", dag.Name, "example")
|
||||
}
|
||||
if len(dag.Steps) != 2 {
|
||||
t.Fatalf("Steps: got %d, want 2", len(dag.Steps))
|
||||
}
|
||||
if dag.Steps[0].Name != "hello" {
|
||||
t.Errorf("Steps[0].Name: got %q, want %q", dag.Steps[0].Name, "hello")
|
||||
}
|
||||
if dag.Steps[1].Name != "list_files" {
|
||||
t.Errorf("Steps[1].Name: got %q, want %q", dag.Steps[1].Name, "list_files")
|
||||
}
|
||||
if len(dag.Steps[1].Depends) != 1 || dag.Steps[1].Depends[0] != "hello" {
|
||||
t.Errorf("Steps[1].Depends: got %v, want [hello]", dag.Steps[1].Depends)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parsea schedule como string y como lista", func(t *testing.T) {
|
||||
// Schedule as string.
|
||||
dataStr := []byte(`
|
||||
name: dag-string-schedule
|
||||
schedule: "0 9 * * 5"
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo ok
|
||||
`)
|
||||
dagStr, err := DagParse(dataStr)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(dagStr.Schedule) != 1 || dagStr.Schedule[0] != "0 9 * * 5" {
|
||||
t.Errorf("Schedule (string): got %v, want [\"0 9 * * 5\"]", dagStr.Schedule)
|
||||
}
|
||||
|
||||
// Schedule as list.
|
||||
dataList := []byte(`
|
||||
name: dag-list-schedule
|
||||
schedule:
|
||||
- "0 9 * * *"
|
||||
- "0 18 * * *"
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo ok
|
||||
`)
|
||||
dagList, err := DagParse(dataList)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(dagList.Schedule) != 2 {
|
||||
t.Fatalf("Schedule (list): got %d items, want 2", len(dagList.Schedule))
|
||||
}
|
||||
if dagList.Schedule[0] != "0 9 * * *" {
|
||||
t.Errorf("Schedule[0]: got %q, want %q", dagList.Schedule[0], "0 9 * * *")
|
||||
}
|
||||
if dagList.Schedule[1] != "0 18 * * *" {
|
||||
t.Errorf("Schedule[1]: got %q, want %q", dagList.Schedule[1], "0 18 * * *")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parsea env en formato lista de maps", func(t *testing.T) {
|
||||
data := []byte(`
|
||||
name: dag-env
|
||||
env:
|
||||
- PROJECT_DIR: /home/lucas/analysis
|
||||
- PYTHON: /home/lucas/.venv/bin/python
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo ${PROJECT_DIR}
|
||||
`)
|
||||
dag, err := DagParse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if dag.Env["PROJECT_DIR"] != "/home/lucas/analysis" {
|
||||
t.Errorf("Env[PROJECT_DIR]: got %q, want %q", dag.Env["PROJECT_DIR"], "/home/lucas/analysis")
|
||||
}
|
||||
if dag.Env["PYTHON"] != "/home/lucas/.venv/bin/python" {
|
||||
t.Errorf("Env[PYTHON]: got %q, want %q", dag.Env["PYTHON"], "/home/lucas/.venv/bin/python")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parsea handler_on y handlers como alias", func(t *testing.T) {
|
||||
// handler_on with single step object.
|
||||
dataHandlerOn := []byte(`
|
||||
name: dag-handler-on
|
||||
handler_on:
|
||||
failure:
|
||||
command: echo "FALLO"
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo hello
|
||||
`)
|
||||
dagHO, err := DagParse(dataHandlerOn)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (handler_on): %v", err)
|
||||
}
|
||||
if len(dagHO.HandlerOn.Failure) != 1 {
|
||||
t.Fatalf("HandlerOn.Failure: got %d steps, want 1", len(dagHO.HandlerOn.Failure))
|
||||
}
|
||||
if dagHO.HandlerOn.Failure[0].Command != `echo "FALLO"` {
|
||||
t.Errorf("HandlerOn.Failure[0].Command: got %q", dagHO.HandlerOn.Failure[0].Command)
|
||||
}
|
||||
|
||||
// handlers alias with list of steps.
|
||||
dataHandlers := []byte(`
|
||||
name: dag-handlers
|
||||
handlers:
|
||||
failure:
|
||||
- name: mark_as_failed
|
||||
command: echo "FAILED"
|
||||
success:
|
||||
- name: notify
|
||||
command: echo "OK"
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo hello
|
||||
`)
|
||||
dagH, err := DagParse(dataHandlers)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error (handlers): %v", err)
|
||||
}
|
||||
if len(dagH.HandlerOn.Failure) != 1 || dagH.HandlerOn.Failure[0].Name != "mark_as_failed" {
|
||||
t.Errorf("HandlerOn.Failure: got %v", dagH.HandlerOn.Failure)
|
||||
}
|
||||
if len(dagH.HandlerOn.Success) != 1 || dagH.HandlerOn.Success[0].Name != "notify" {
|
||||
t.Errorf("HandlerOn.Success: got %v", dagH.HandlerOn.Success)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parsea continue_on y working_dir a nivel step", func(t *testing.T) {
|
||||
data := []byte(`
|
||||
name: dag-step-options
|
||||
steps:
|
||||
- id: ingest
|
||||
description: Procesar archivos
|
||||
working_dir: /home/lucas/project
|
||||
command: ./bin/ingest
|
||||
continue_on:
|
||||
failure: true
|
||||
- id: informe
|
||||
command: python /tmp/script.py
|
||||
depends: [ingest]
|
||||
`)
|
||||
dag, err := DagParse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(dag.Steps) != 2 {
|
||||
t.Fatalf("Steps: got %d, want 2", len(dag.Steps))
|
||||
}
|
||||
step0 := dag.Steps[0]
|
||||
if step0.ID != "ingest" {
|
||||
t.Errorf("Steps[0].ID: got %q, want %q", step0.ID, "ingest")
|
||||
}
|
||||
if step0.Dir != "/home/lucas/project" {
|
||||
t.Errorf("Steps[0].Dir: got %q, want %q", step0.Dir, "/home/lucas/project")
|
||||
}
|
||||
if !step0.ContinueOn.Failure {
|
||||
t.Errorf("Steps[0].ContinueOn.Failure: got false, want true")
|
||||
}
|
||||
step1 := dag.Steps[1]
|
||||
if len(step1.Depends) != 1 || step1.Depends[0] != "ingest" {
|
||||
t.Errorf("Steps[1].Depends: got %v, want [ingest]", step1.Depends)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("parsea type graph", func(t *testing.T) {
|
||||
data := []byte(`
|
||||
name: dag-graph
|
||||
type: graph
|
||||
tags: [finanzas, semanal]
|
||||
steps:
|
||||
- name: step1
|
||||
command: echo a
|
||||
- name: step2
|
||||
command: echo b
|
||||
depends: [step1]
|
||||
`)
|
||||
dag, err := DagParse(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if dag.Type != "graph" {
|
||||
t.Errorf("Type: got %q, want %q", dag.Type, "graph")
|
||||
}
|
||||
if len(dag.Tags) != 2 {
|
||||
t.Errorf("Tags: got %v, want [finanzas semanal]", dag.Tags)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DagResolveEnv resolves environment variable references in a DagDefinition.
|
||||
// It parses environ (KEY=VALUE pairs from os.Environ()), merges with dag.Env
|
||||
// (dag.Env takes precedence), and substitutes ${VAR} and $VAR in Command,
|
||||
// Script, Dir, Args, and Env values of each step. Also substitutes in
|
||||
// DagDefinition.WorkingDir. Returns a new DagDefinition without mutating the original.
|
||||
func DagResolveEnv(dag DagDefinition, environ []string) DagDefinition {
|
||||
// Parse environ into a base map.
|
||||
base := parseEnviron(environ)
|
||||
|
||||
// Merge dag.Env over base (dag.Env has precedence).
|
||||
dagEnv := mergeMaps(base, dag.Env)
|
||||
|
||||
// Build new DagDefinition (shallow copy, then replace fields).
|
||||
result := dag
|
||||
|
||||
// Substitute in WorkingDir.
|
||||
result.WorkingDir = substitute(dag.WorkingDir, dagEnv)
|
||||
|
||||
// Substitute in Env values of the DAG itself.
|
||||
result.Env = substituteMap(dag.Env, dagEnv)
|
||||
|
||||
// Resolve steps.
|
||||
resolvedSteps := make([]DagStep, len(dag.Steps))
|
||||
for i, step := range dag.Steps {
|
||||
// Merge dag env with step env (step env has precedence).
|
||||
stepEnv := mergeMaps(dagEnv, step.Env)
|
||||
|
||||
rs := step
|
||||
rs.Command = substitute(step.Command, stepEnv)
|
||||
rs.Script = substitute(step.Script, stepEnv)
|
||||
rs.Dir = substitute(step.Dir, stepEnv)
|
||||
|
||||
if len(step.Args) > 0 {
|
||||
args := make([]string, len(step.Args))
|
||||
for j, arg := range step.Args {
|
||||
args[j] = substitute(arg, stepEnv)
|
||||
}
|
||||
rs.Args = args
|
||||
}
|
||||
|
||||
rs.Env = substituteMap(step.Env, stepEnv)
|
||||
resolvedSteps[i] = rs
|
||||
}
|
||||
result.Steps = resolvedSteps
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// parseEnviron parses a slice of "KEY=VALUE" strings into a map.
|
||||
func parseEnviron(environ []string) map[string]string {
|
||||
m := make(map[string]string, len(environ))
|
||||
for _, kv := range environ {
|
||||
idx := strings.IndexByte(kv, '=')
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
m[kv[:idx]] = kv[idx+1:]
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// mergeMaps returns a new map with all entries from base, then overlaid with overlay.
|
||||
func mergeMaps(base, overlay map[string]string) map[string]string {
|
||||
result := make(map[string]string, len(base)+len(overlay))
|
||||
for k, v := range base {
|
||||
result[k] = v
|
||||
}
|
||||
for k, v := range overlay {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// substituteMap applies substitute to all values in a map.
|
||||
func substituteMap(m map[string]string, env map[string]string) map[string]string {
|
||||
if len(m) == 0 {
|
||||
return m
|
||||
}
|
||||
result := make(map[string]string, len(m))
|
||||
for k, v := range m {
|
||||
result[k] = substitute(v, env)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// substitute replaces ${VAR} and $VAR occurrences in s using the env map.
|
||||
func substitute(s string, env map[string]string) string {
|
||||
if s == "" || !strings.Contains(s, "$") {
|
||||
return s
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
i := 0
|
||||
for i < len(s) {
|
||||
if s[i] != '$' {
|
||||
b.WriteByte(s[i])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
// Found '$'
|
||||
i++
|
||||
if i >= len(s) {
|
||||
b.WriteByte('$')
|
||||
break
|
||||
}
|
||||
|
||||
if s[i] == '{' {
|
||||
// ${VAR} form.
|
||||
i++
|
||||
end := strings.IndexByte(s[i:], '}')
|
||||
if end < 0 {
|
||||
// No closing brace — write as-is.
|
||||
b.WriteString("${")
|
||||
continue
|
||||
}
|
||||
varName := s[i : i+end]
|
||||
i += end + 1
|
||||
if val, ok := env[varName]; ok {
|
||||
b.WriteString(val)
|
||||
} else {
|
||||
b.WriteString("${")
|
||||
b.WriteString(varName)
|
||||
b.WriteByte('}')
|
||||
}
|
||||
} else if isEnvVarChar(s[i]) {
|
||||
// $VAR form (bare variable name).
|
||||
start := i
|
||||
for i < len(s) && isEnvVarChar(s[i]) {
|
||||
i++
|
||||
}
|
||||
varName := s[start:i]
|
||||
if val, ok := env[varName]; ok {
|
||||
b.WriteString(val)
|
||||
} else {
|
||||
b.WriteByte('$')
|
||||
b.WriteString(varName)
|
||||
}
|
||||
} else {
|
||||
// Not a valid variable start — keep '$' and continue.
|
||||
b.WriteByte('$')
|
||||
}
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// isEnvVarChar returns true for characters valid in environment variable names.
|
||||
func isEnvVarChar(c byte) bool {
|
||||
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '_'
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
name: dag_resolve_env
|
||||
kind: function
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
purity: pure
|
||||
signature: "func DagResolveEnv(dag DagDefinition, environ []string) DagDefinition"
|
||||
description: "Resuelve referencias a variables de entorno (${VAR} y $VAR) en un DagDefinition. Mergea environ del sistema con dag.Env (dag.Env tiene precedencia), y sustituye en Command, Script, Dir, Args y Env values de cada step. No muta el DagDefinition original."
|
||||
tags: [dag, env, substitution, variable-expansion, pure]
|
||||
uses_functions: []
|
||||
uses_types: [dag_definition_go_core, dag_step_go_core]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: ""
|
||||
imports: [strings]
|
||||
params:
|
||||
- name: dag
|
||||
desc: "DagDefinition parseado con posibles referencias ${VAR} en sus campos"
|
||||
- name: environ
|
||||
desc: "lista de strings KEY=VALUE del entorno del sistema (tipicamente os.Environ())"
|
||||
output: "nuevo DagDefinition con todas las referencias de variables sustituidas por sus valores"
|
||||
tested: false
|
||||
tests: []
|
||||
test_file_path: ""
|
||||
file_path: "functions/core/dag_resolve_env.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
dag, _ := DagParse(yamlData)
|
||||
resolved := DagResolveEnv(dag, os.Environ())
|
||||
// resolved.Steps[0].Command tiene ${PROJECT_DIR} sustituido
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion pura. No muta el DagDefinition de entrada — retorna una copia con los campos resueltos. Precedencia de variables: environ base < dag.Env < step.Env. Sustituye tanto ${VAR} (con llaves) como $VAR (bare). Si una variable no existe en el entorno, se deja sin sustituir. Los campos sustituidos son: DagDefinition.WorkingDir, Env values del DAG, y por step: Command, Script, Dir, Args, Env values.
|
||||
@@ -0,0 +1,78 @@
|
||||
package core
|
||||
|
||||
import "fmt"
|
||||
|
||||
// DagTopoSort performs a topological sort of DAG steps using Kahn's algorithm.
|
||||
// Returns levels where each level contains steps that can run in parallel.
|
||||
// Returns an error if a dependency cycle is detected.
|
||||
func DagTopoSort(steps []DagStep) ([][]DagStep, error) {
|
||||
if len(steps) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build index: ref -> DagStep.
|
||||
index := make(map[string]DagStep, len(steps))
|
||||
for _, s := range steps {
|
||||
index[stepRef(s)] = s
|
||||
}
|
||||
|
||||
// Compute in-degree for each step.
|
||||
inDegree := make(map[string]int, len(steps))
|
||||
for _, s := range steps {
|
||||
ref := stepRef(s)
|
||||
if _, ok := inDegree[ref]; !ok {
|
||||
inDegree[ref] = 0
|
||||
}
|
||||
for range s.Depends {
|
||||
inDegree[ref]++ // ref depends on a dep, so ref gets +1
|
||||
}
|
||||
}
|
||||
|
||||
// Build adjacency list: dep -> list of steps that depend on dep.
|
||||
adj := make(map[string][]string, len(steps))
|
||||
for _, s := range steps {
|
||||
ref := stepRef(s)
|
||||
for _, dep := range s.Depends {
|
||||
adj[dep] = append(adj[dep], ref)
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize queue with steps that have in-degree 0.
|
||||
var queue []string
|
||||
for _, s := range steps {
|
||||
ref := stepRef(s)
|
||||
if inDegree[ref] == 0 {
|
||||
queue = append(queue, ref)
|
||||
}
|
||||
}
|
||||
|
||||
var levels [][]DagStep
|
||||
processed := 0
|
||||
|
||||
for len(queue) > 0 {
|
||||
// The current queue forms a parallel level.
|
||||
level := make([]DagStep, 0, len(queue))
|
||||
nextQueue := []string{}
|
||||
|
||||
for _, ref := range queue {
|
||||
level = append(level, index[ref])
|
||||
processed++
|
||||
// Reduce in-degree for dependents.
|
||||
for _, dependent := range adj[ref] {
|
||||
inDegree[dependent]--
|
||||
if inDegree[dependent] == 0 {
|
||||
nextQueue = append(nextQueue, dependent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
levels = append(levels, level)
|
||||
queue = nextQueue
|
||||
}
|
||||
|
||||
if processed != len(steps) {
|
||||
return nil, fmt.Errorf("dag_topo_sort: cycle detected (%d of %d steps processed)", processed, len(steps))
|
||||
}
|
||||
|
||||
return levels, nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
---
|
||||
name: dag_topo_sort
|
||||
kind: function
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
purity: pure
|
||||
signature: "func DagTopoSort(steps []DagStep) ([][]DagStep, error)"
|
||||
description: "Ordenamiento topologico de steps de un DAG usando el algoritmo de Kahn. Retorna niveles donde cada nivel contiene steps que pueden ejecutarse en paralelo. Detecta ciclos y retorna error si los hay."
|
||||
tags: [dag, topological-sort, graph, kahn, pure]
|
||||
uses_functions: []
|
||||
uses_types: [dag_step_go_core]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: ""
|
||||
imports: [fmt]
|
||||
params:
|
||||
- name: steps
|
||||
desc: "lista de DagStep con sus dependencias definidas en el campo Depends"
|
||||
output: "slice de niveles donde cada nivel es un slice de DagStep que pueden ejecutarse en paralelo; error si hay ciclo"
|
||||
tested: true
|
||||
tests:
|
||||
- "cadena lineal A->B->C produce tres niveles"
|
||||
- "diamond A->B A->C B->D C->D produce cuatro niveles"
|
||||
- "steps paralelos sin depends produce un solo nivel"
|
||||
- "ciclo A->B->A retorna error"
|
||||
test_file_path: "functions/core/dag_topo_sort_test.go"
|
||||
file_path: "functions/core/dag_topo_sort.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
steps := []DagStep{
|
||||
{Name: "a"},
|
||||
{Name: "b", Depends: []string{"a"}},
|
||||
{Name: "c", Depends: []string{"a"}},
|
||||
{Name: "d", Depends: []string{"b", "c"}},
|
||||
}
|
||||
levels, err := DagTopoSort(steps)
|
||||
// levels[0] = [a]
|
||||
// levels[1] = [b, c] (paralelo)
|
||||
// levels[2] = [d]
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion pura. Implementa Kahn's algorithm: calcula in-degree inicial, pone en cola los steps con in-degree 0, extrae un nivel completo por iteracion y reduce el in-degree de los dependientes. Si al terminar hay steps sin procesar, hay un ciclo. El orden dentro de cada nivel no esta garantizado — depende del orden del slice de entrada.
|
||||
@@ -0,0 +1,95 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDagTopoSort(t *testing.T) {
|
||||
t.Run("cadena lineal A->B->C produce tres niveles", func(t *testing.T) {
|
||||
steps := []DagStep{
|
||||
{Name: "a"},
|
||||
{Name: "b", Depends: []string{"a"}},
|
||||
{Name: "c", Depends: []string{"b"}},
|
||||
}
|
||||
levels, err := DagTopoSort(steps)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(levels) != 3 {
|
||||
t.Fatalf("levels: got %d, want 3", len(levels))
|
||||
}
|
||||
if len(levels[0]) != 1 || levels[0][0].Name != "a" {
|
||||
t.Errorf("levels[0]: got %v, want [a]", stepNames(levels[0]))
|
||||
}
|
||||
if len(levels[1]) != 1 || levels[1][0].Name != "b" {
|
||||
t.Errorf("levels[1]: got %v, want [b]", stepNames(levels[1]))
|
||||
}
|
||||
if len(levels[2]) != 1 || levels[2][0].Name != "c" {
|
||||
t.Errorf("levels[2]: got %v, want [c]", stepNames(levels[2]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("diamond A->B A->C B->D C->D produce cuatro niveles", func(t *testing.T) {
|
||||
steps := []DagStep{
|
||||
{Name: "a"},
|
||||
{Name: "b", Depends: []string{"a"}},
|
||||
{Name: "c", Depends: []string{"a"}},
|
||||
{Name: "d", Depends: []string{"b", "c"}},
|
||||
}
|
||||
levels, err := DagTopoSort(steps)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
// Expected: level0=[a], level1=[b,c], level2=[d]
|
||||
if len(levels) != 3 {
|
||||
t.Fatalf("levels: got %d, want 3", len(levels))
|
||||
}
|
||||
if len(levels[0]) != 1 || levels[0][0].Name != "a" {
|
||||
t.Errorf("levels[0]: got %v, want [a]", stepNames(levels[0]))
|
||||
}
|
||||
if len(levels[1]) != 2 {
|
||||
t.Errorf("levels[1]: got %v, want 2 steps", stepNames(levels[1]))
|
||||
}
|
||||
if len(levels[2]) != 1 || levels[2][0].Name != "d" {
|
||||
t.Errorf("levels[2]: got %v, want [d]", stepNames(levels[2]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("steps paralelos sin depends produce un solo nivel", func(t *testing.T) {
|
||||
steps := []DagStep{
|
||||
{Name: "x"},
|
||||
{Name: "y"},
|
||||
{Name: "z"},
|
||||
}
|
||||
levels, err := DagTopoSort(steps)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(levels) != 1 {
|
||||
t.Fatalf("levels: got %d, want 1", len(levels))
|
||||
}
|
||||
if len(levels[0]) != 3 {
|
||||
t.Errorf("levels[0]: got %d steps, want 3", len(levels[0]))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ciclo A->B->A retorna error", func(t *testing.T) {
|
||||
steps := []DagStep{
|
||||
{Name: "a", Depends: []string{"b"}},
|
||||
{Name: "b", Depends: []string{"a"}},
|
||||
}
|
||||
_, err := DagTopoSort(steps)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for cycle, got nil")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// stepNames extracts step names for readable test output.
|
||||
func stepNames(steps []DagStep) []string {
|
||||
names := make([]string, len(steps))
|
||||
for i, s := range steps {
|
||||
names[i] = stepRef(s)
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package core
|
||||
|
||||
import "fmt"
|
||||
|
||||
// DagValidate validates a DagDefinition for structural correctness.
|
||||
// Checks: steps have name/ID, no duplicate names/IDs, all depends reference
|
||||
// existing steps, no dependency cycles. On success, computes topological levels.
|
||||
// Returns warnings for steps with both command and script set.
|
||||
func DagValidate(dag DagDefinition) DagValidationResult {
|
||||
result := DagValidationResult{Valid: true}
|
||||
|
||||
// Build name/ID sets and check for missing identifiers and duplicates.
|
||||
seen := make(map[string]bool)
|
||||
for i, step := range dag.Steps {
|
||||
ref := stepRef(step)
|
||||
if ref == "" {
|
||||
result.Errors = append(result.Errors,
|
||||
fmt.Sprintf("step[%d]: must have name or id", i))
|
||||
result.Valid = false
|
||||
continue
|
||||
}
|
||||
if seen[ref] {
|
||||
result.Errors = append(result.Errors,
|
||||
fmt.Sprintf("step[%d]: duplicate name/id %q", i, ref))
|
||||
result.Valid = false
|
||||
}
|
||||
seen[ref] = true
|
||||
|
||||
// Warning: command and script both set.
|
||||
if step.Command != "" && step.Script != "" {
|
||||
result.Warnings = append(result.Warnings,
|
||||
fmt.Sprintf("step %q: has both command and script", ref))
|
||||
}
|
||||
}
|
||||
|
||||
if !result.Valid {
|
||||
return result
|
||||
}
|
||||
|
||||
// Check that all depends reference existing steps.
|
||||
for _, step := range dag.Steps {
|
||||
for _, dep := range step.Depends {
|
||||
if !seen[dep] {
|
||||
result.Errors = append(result.Errors,
|
||||
fmt.Sprintf("step %q: depends on unknown step %q", stepRef(step), dep))
|
||||
result.Valid = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !result.Valid {
|
||||
return result
|
||||
}
|
||||
|
||||
// Topological sort with Kahn's — detects cycles and computes levels.
|
||||
levels, err := DagTopoSort(dag.Steps)
|
||||
if err != nil {
|
||||
result.Errors = append(result.Errors, fmt.Sprintf("cycle detected: %v", err))
|
||||
result.Valid = false
|
||||
return result
|
||||
}
|
||||
|
||||
// Convert [][]DagStep to [][]string for the result.
|
||||
strLevels := make([][]string, len(levels))
|
||||
for i, level := range levels {
|
||||
refs := make([]string, len(level))
|
||||
for j, s := range level {
|
||||
refs[j] = stepRef(s)
|
||||
}
|
||||
strLevels[i] = refs
|
||||
}
|
||||
result.Levels = strLevels
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// stepRef returns the canonical reference for a step (ID preferred, then Name).
|
||||
func stepRef(s DagStep) string {
|
||||
if s.ID != "" {
|
||||
return s.ID
|
||||
}
|
||||
return s.Name
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
---
|
||||
name: dag_validate
|
||||
kind: function
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
purity: pure
|
||||
signature: "func DagValidate(dag DagDefinition) DagValidationResult"
|
||||
description: "Valida un DagDefinition para correcto uso estructural. Verifica que cada step tenga nombre o ID, que no haya duplicados, que todos los depends referencien steps existentes y que no haya ciclos (algoritmo de Kahn). Si el DAG es valido, calcula los niveles topologicos."
|
||||
tags: [dag, validation, workflow, graph, pure]
|
||||
uses_functions: [dag_topo_sort_go_core]
|
||||
uses_types: [dag_definition_go_core, dag_validation_result_go_core]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: ""
|
||||
imports: [fmt]
|
||||
params:
|
||||
- name: dag
|
||||
desc: "DagDefinition parseado y a validar"
|
||||
output: "DagValidationResult con Valid=true si no hay errores, Errors con los problemas encontrados, Warnings con avisos no fatales, y Levels con los niveles topologicos si el DAG es valido"
|
||||
tested: true
|
||||
tests:
|
||||
- "DAG valido retorna Valid true y levels calculados"
|
||||
- "step sin nombre retorna error"
|
||||
- "depends a step inexistente retorna error"
|
||||
- "ciclo en dependencias retorna error"
|
||||
test_file_path: "functions/core/dag_validate_test.go"
|
||||
file_path: "functions/core/dag_validate.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
dag, _ := DagParse(yamlData)
|
||||
res := DagValidate(dag)
|
||||
if !res.Valid {
|
||||
for _, e := range res.Errors {
|
||||
fmt.Println("ERROR:", e)
|
||||
}
|
||||
}
|
||||
// res.Levels = [["step-a"], ["step-b", "step-c"], ["step-d"]]
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion pura. No modifica el DagDefinition de entrada. El calculo de niveles topologicos se delega a DagTopoSort para mantener la separacion de responsabilidades. Un warning (command+script simultaneos) no invalida el DAG.
|
||||
@@ -0,0 +1,113 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDagValidate(t *testing.T) {
|
||||
t.Run("DAG valido retorna Valid true y levels calculados", func(t *testing.T) {
|
||||
dag := DagDefinition{
|
||||
Name: "valid-dag",
|
||||
Steps: []DagStep{
|
||||
{Name: "a", Command: "echo a"},
|
||||
{Name: "b", Command: "echo b", Depends: []string{"a"}},
|
||||
{Name: "c", Command: "echo c", Depends: []string{"a"}},
|
||||
{Name: "d", Command: "echo d", Depends: []string{"b", "c"}},
|
||||
},
|
||||
}
|
||||
res := DagValidate(dag)
|
||||
if !res.Valid {
|
||||
t.Errorf("Valid: got false, want true. Errors: %v", res.Errors)
|
||||
}
|
||||
if len(res.Errors) != 0 {
|
||||
t.Errorf("Errors: got %v, want empty", res.Errors)
|
||||
}
|
||||
// Should have 3 levels: [a], [b,c], [d]
|
||||
if len(res.Levels) != 3 {
|
||||
t.Errorf("Levels: got %d, want 3. Levels: %v", len(res.Levels), res.Levels)
|
||||
}
|
||||
if len(res.Levels[0]) != 1 || res.Levels[0][0] != "a" {
|
||||
t.Errorf("Levels[0]: got %v, want [a]", res.Levels[0])
|
||||
}
|
||||
if len(res.Levels[2]) != 1 || res.Levels[2][0] != "d" {
|
||||
t.Errorf("Levels[2]: got %v, want [d]", res.Levels[2])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("step sin nombre retorna error", func(t *testing.T) {
|
||||
dag := DagDefinition{
|
||||
Name: "bad-dag",
|
||||
Steps: []DagStep{
|
||||
{Command: "echo hello"}, // no name, no ID
|
||||
},
|
||||
}
|
||||
res := DagValidate(dag)
|
||||
if res.Valid {
|
||||
t.Error("Valid: got true, want false")
|
||||
}
|
||||
if len(res.Errors) == 0 {
|
||||
t.Error("Errors: got empty, want at least one error")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("depends a step inexistente retorna error", func(t *testing.T) {
|
||||
dag := DagDefinition{
|
||||
Name: "bad-deps-dag",
|
||||
Steps: []DagStep{
|
||||
{Name: "step1", Command: "echo ok"},
|
||||
{Name: "step2", Command: "echo ok", Depends: []string{"nonexistent"}},
|
||||
},
|
||||
}
|
||||
res := DagValidate(dag)
|
||||
if res.Valid {
|
||||
t.Error("Valid: got true, want false")
|
||||
}
|
||||
found := false
|
||||
for _, e := range res.Errors {
|
||||
if containsStr(e, "nonexistent") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Errors should mention 'nonexistent', got: %v", res.Errors)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ciclo en dependencias retorna error", func(t *testing.T) {
|
||||
dag := DagDefinition{
|
||||
Name: "cyclic-dag",
|
||||
Steps: []DagStep{
|
||||
{Name: "a", Command: "echo a", Depends: []string{"b"}},
|
||||
{Name: "b", Command: "echo b", Depends: []string{"a"}},
|
||||
},
|
||||
}
|
||||
res := DagValidate(dag)
|
||||
if res.Valid {
|
||||
t.Error("Valid: got true, want false")
|
||||
}
|
||||
found := false
|
||||
for _, e := range res.Errors {
|
||||
if containsStr(e, "cycle") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Errors should mention 'cycle', got: %v", res.Errors)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// containsStr returns true if s contains substr.
|
||||
func containsStr(s, substr string) bool {
|
||||
return len(s) >= len(substr) && (s == substr || len(substr) == 0 ||
|
||||
func() bool {
|
||||
for i := 0; i <= len(s)-len(substr); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}())
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package infra
|
||||
|
||||
import "time"
|
||||
|
||||
// DagRun represents one execution of a DAG workflow.
|
||||
type DagRun struct {
|
||||
ID string
|
||||
DagName string
|
||||
DagPath string
|
||||
Status string // pending, running, success, failed, cancelled
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Trigger string // manual, cron, api
|
||||
Error string
|
||||
}
|
||||
|
||||
// DagStepResult represents the outcome of one step within a DagRun.
|
||||
type DagStepResult struct {
|
||||
ID string
|
||||
RunID string
|
||||
StepName string
|
||||
Status string // pending, running, success, failed, skipped
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
DurationMs int64
|
||||
Error string
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessHandle represents a running subprocess with output buffers.
|
||||
type ProcessHandle struct {
|
||||
Cmd *exec.Cmd
|
||||
Pid int
|
||||
StartTime time.Time
|
||||
Dir string
|
||||
stdout *bytes.Buffer
|
||||
stderr *bytes.Buffer
|
||||
}
|
||||
|
||||
// ProcessResult contains the outcome of a completed subprocess.
|
||||
type ProcessResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
DurationMs int64
|
||||
Killed bool
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessKill sends SIGTERM to the process group of handle, then waits up to
|
||||
// graceSec seconds for the process to exit. If it is still alive after the
|
||||
// grace period, SIGKILL is sent. Returns an error only if the signal could not
|
||||
// be delivered (e.g. the process group does not exist).
|
||||
func ProcessKill(handle *ProcessHandle, graceSec int) error {
|
||||
// Send SIGTERM to the process group (negative pid targets the group).
|
||||
if err := syscall.Kill(-handle.Pid, syscall.SIGTERM); err != nil {
|
||||
// ESRCH means the process is already gone — not an error from our view.
|
||||
if err != syscall.ESRCH {
|
||||
return fmt.Errorf("process_kill: sigterm: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Poll until the process exits or the grace period expires.
|
||||
deadline := time.Now().Add(time.Duration(graceSec) * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
// Check if process has exited by sending signal 0 (no-op).
|
||||
err := syscall.Kill(-handle.Pid, 0)
|
||||
if err == syscall.ESRCH {
|
||||
// Process group is gone.
|
||||
return nil
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Still alive after grace period — escalate to SIGKILL.
|
||||
if err := syscall.Kill(-handle.Pid, syscall.SIGKILL); err != nil {
|
||||
if err != syscall.ESRCH {
|
||||
return fmt.Errorf("process_kill: sigkill: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
---
|
||||
name: process_kill
|
||||
kind: function
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "func ProcessKill(handle *ProcessHandle, graceSec int) error"
|
||||
description: "Termina un subproceso enviando SIGTERM al process group. Espera hasta graceSec segundos a que el proceso muera voluntariamente. Si sigue vivo, envia SIGKILL. Retorna error solo si la senal no pudo entregarse."
|
||||
tags: [process, subprocess, kill, signal, sigterm, sigkill, infra]
|
||||
uses_functions: []
|
||||
uses_types: [process_handle_go_infra]
|
||||
returns: []
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: [fmt, syscall, time]
|
||||
params:
|
||||
- name: handle
|
||||
desc: "handle del proceso lanzado por ProcessSpawn"
|
||||
- name: graceSec
|
||||
desc: "segundos de gracia entre SIGTERM y SIGKILL; 0 envia SIGKILL inmediatamente"
|
||||
output: "nil si el proceso fue terminado correctamente; error si la senal no pudo entregarse"
|
||||
tested: true
|
||||
tests:
|
||||
- "kill process"
|
||||
test_file_path: "functions/infra/process_spawn_test.go"
|
||||
file_path: "functions/infra/process_kill.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
h, err := ProcessSpawn("sleep 300", "", nil, "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// Dar 3 segundos de gracia antes de SIGKILL
|
||||
if err := ProcessKill(h, 3); err != nil {
|
||||
log.Printf("kill failed: %v", err)
|
||||
}
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion impura: envia senales al sistema operativo. Usa -handle.Pid (negativo) para direccionar el process group completo, matando tanto al proceso principal como a sus hijos. ESRCH se ignora porque significa que el proceso ya murio, lo cual es el objetivo deseado. Comprueba si el proceso sigue vivo con signal 0 (kill -0) cada 100ms durante el grace period.
|
||||
@@ -0,0 +1,74 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessSpawn launches a subprocess using the given shell.
|
||||
// If shell is empty, "sh" is used. If command contains newlines it is treated
|
||||
// as a multi-line script: the content is written to a temp file and executed
|
||||
// with `shell <tempfile>`. Otherwise it is executed with `shell -c <command>`.
|
||||
// dir sets the working directory (empty = inherit). env sets the environment
|
||||
// (nil = inherit parent env). The process group is created with Setpgid so
|
||||
// that ProcessKill can target the whole group.
|
||||
func ProcessSpawn(command string, dir string, env []string, shell string) (*ProcessHandle, error) {
|
||||
if shell == "" {
|
||||
shell = "sh"
|
||||
}
|
||||
|
||||
var cmd *exec.Cmd
|
||||
|
||||
if strings.Contains(command, "\n") {
|
||||
// Multi-line script: write to a temp file and execute it.
|
||||
tmp, err := os.CreateTemp("", "fn-proc-*.sh")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("process_spawn: create temp file: %w", err)
|
||||
}
|
||||
if _, err := tmp.WriteString(command); err != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
return nil, fmt.Errorf("process_spawn: write temp file: %w", err)
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmp.Name())
|
||||
return nil, fmt.Errorf("process_spawn: close temp file: %w", err)
|
||||
}
|
||||
cmd = exec.Command(shell, tmp.Name())
|
||||
} else {
|
||||
cmd = exec.Command(shell, "-c", command)
|
||||
}
|
||||
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
if len(env) > 0 {
|
||||
cmd.Env = env
|
||||
}
|
||||
|
||||
// New process group so we can kill all children as a group.
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
// Use buffers instead of pipes to avoid race between Wait() and ReadAll().
|
||||
var stdoutBuf, stderrBuf bytes.Buffer
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
start := time.Now()
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("process_spawn: start: %w", err)
|
||||
}
|
||||
|
||||
return &ProcessHandle{
|
||||
Cmd: cmd,
|
||||
Pid: cmd.Process.Pid,
|
||||
StartTime: start,
|
||||
Dir: dir,
|
||||
stdout: &stdoutBuf,
|
||||
stderr: &stderrBuf,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
---
|
||||
name: process_spawn
|
||||
kind: function
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "func ProcessSpawn(command string, dir string, env []string, shell string) (*ProcessHandle, error)"
|
||||
description: "Lanza un subproceso usando el shell indicado. Si shell esta vacio usa 'sh'. Comandos con newlines se tratan como scripts multilinea (se escriben a un archivo temporal). Configura un process group propio (Setpgid) para poder matar todos los hijos con ProcessKill. Captura stdout y stderr via pipes."
|
||||
tags: [process, subprocess, spawn, exec, shell, infra]
|
||||
uses_functions: []
|
||||
uses_types: [process_handle_go_infra]
|
||||
returns: [process_handle_go_infra]
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: [fmt, os, os/exec, strings, syscall, time]
|
||||
params:
|
||||
- name: command
|
||||
desc: "comando shell a ejecutar; si contiene newlines se trata como script multilinea"
|
||||
- name: dir
|
||||
desc: "directorio de trabajo del proceso hijo; vacio hereda el del proceso padre"
|
||||
- name: env
|
||||
desc: "variables de entorno en formato KEY=VALUE; nil hereda el entorno del proceso padre"
|
||||
- name: shell
|
||||
desc: "interprete shell a usar (sh, bash, zsh); vacio usa 'sh'"
|
||||
output: "handle del proceso lanzado con Cmd, Pid, StartTime, Dir y los pipes de I/O"
|
||||
tested: true
|
||||
tests:
|
||||
- "spawn and wait echo"
|
||||
- "spawn with timeout kills"
|
||||
- "spawn with env"
|
||||
- "spawn script"
|
||||
- "spawn with working dir"
|
||||
- "kill process"
|
||||
test_file_path: "functions/infra/process_spawn_test.go"
|
||||
file_path: "functions/infra/process_spawn.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
h, err := ProcessSpawn("echo hello", "", nil, "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
res, err := ProcessWait(h, 10)
|
||||
fmt.Println(res.Stdout) // "hello\n"
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion impura: hace I/O (crea archivo temporal para scripts, lanza proceso). El process group (Setpgid=true) permite a ProcessKill enviar senales al grupo completo con -Pid, afectando a todos los hijos del proceso lanzado. Para scripts multilinea el archivo temporal queda en el directorio temporal del OS y no se limpia automaticamente.
|
||||
@@ -0,0 +1,107 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProcessSpawn(t *testing.T) {
|
||||
t.Run("spawn and wait echo", func(t *testing.T) {
|
||||
h, err := ProcessSpawn("echo hello", "", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
res, err := ProcessWait(h, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if res.ExitCode != 0 {
|
||||
t.Errorf("exit code: got %d, want 0", res.ExitCode)
|
||||
}
|
||||
if !strings.Contains(res.Stdout, "hello") {
|
||||
t.Errorf("stdout: got %q, want it to contain 'hello'", res.Stdout)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spawn with timeout kills", func(t *testing.T) {
|
||||
h, err := ProcessSpawn("sleep 60", "", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
res, err := ProcessWait(h, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if !res.Killed {
|
||||
t.Errorf("killed: got false, want true")
|
||||
}
|
||||
if res.ExitCode == 0 {
|
||||
t.Errorf("exit code: got 0, want != 0 after kill")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spawn with env", func(t *testing.T) {
|
||||
h, err := ProcessSpawn("echo $TEST_VAR", "", []string{"PATH=/usr/bin:/bin", "TEST_VAR=hello123"}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
res, err := ProcessWait(h, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if !strings.Contains(res.Stdout, "hello123") {
|
||||
t.Errorf("stdout: got %q, want it to contain 'hello123'", res.Stdout)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spawn script", func(t *testing.T) {
|
||||
script := "#!/bin/sh\necho line1\necho line2"
|
||||
h, err := ProcessSpawn(script, "", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
res, err := ProcessWait(h, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if !strings.Contains(res.Stdout, "line1") {
|
||||
t.Errorf("stdout: got %q, want it to contain 'line1'", res.Stdout)
|
||||
}
|
||||
if !strings.Contains(res.Stdout, "line2") {
|
||||
t.Errorf("stdout: got %q, want it to contain 'line2'", res.Stdout)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("spawn with working dir", func(t *testing.T) {
|
||||
h, err := ProcessSpawn("pwd", "/tmp", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
res, err := ProcessWait(h, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("wait: %v", err)
|
||||
}
|
||||
if !strings.Contains(res.Stdout, "/tmp") {
|
||||
t.Errorf("stdout: got %q, want it to contain '/tmp'", res.Stdout)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("kill process", func(t *testing.T) {
|
||||
h, err := ProcessSpawn("sleep 60", "", nil, "")
|
||||
if err != nil {
|
||||
t.Fatalf("spawn: %v", err)
|
||||
}
|
||||
if err := ProcessKill(h, 1); err != nil {
|
||||
t.Fatalf("kill: %v", err)
|
||||
}
|
||||
// After kill, Wait should unblock quickly.
|
||||
_ = h.Cmd.Wait()
|
||||
state := h.Cmd.ProcessState
|
||||
if state == nil {
|
||||
t.Fatal("process state is nil after kill+wait")
|
||||
}
|
||||
if state.ExitCode() == 0 {
|
||||
t.Errorf("exit code: got 0 after kill, want non-zero")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessWait waits for a subprocess to finish and collects its output.
|
||||
// If timeoutSec > 0 and the process has not exited by then, ProcessKill is
|
||||
// called with graceSec=5 and the result is marked Killed=true.
|
||||
func ProcessWait(handle *ProcessHandle, timeoutSec int) (ProcessResult, error) {
|
||||
// Wait for the process in a goroutine.
|
||||
waitCh := make(chan error, 1)
|
||||
go func() {
|
||||
waitCh <- handle.Cmd.Wait()
|
||||
}()
|
||||
|
||||
killed := false
|
||||
|
||||
if timeoutSec > 0 {
|
||||
timer := time.NewTimer(time.Duration(timeoutSec) * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
// Timeout exceeded — kill the process group.
|
||||
_ = ProcessKill(handle, 5)
|
||||
killed = true
|
||||
<-waitCh
|
||||
case <-waitCh:
|
||||
}
|
||||
} else {
|
||||
<-waitCh
|
||||
}
|
||||
|
||||
// After Wait() returns, buffers are safe to read.
|
||||
exitCode := 0
|
||||
if handle.Cmd.ProcessState != nil {
|
||||
exitCode = handle.Cmd.ProcessState.ExitCode()
|
||||
} else if killed {
|
||||
exitCode = -1
|
||||
}
|
||||
|
||||
duration := time.Since(handle.StartTime)
|
||||
|
||||
return ProcessResult{
|
||||
ExitCode: exitCode,
|
||||
Stdout: handle.stdout.String(),
|
||||
Stderr: handle.stderr.String(),
|
||||
DurationMs: duration.Milliseconds(),
|
||||
Killed: killed,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
name: process_wait
|
||||
kind: function
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
purity: impure
|
||||
signature: "func ProcessWait(handle *ProcessHandle, timeoutSec int) (ProcessResult, error)"
|
||||
description: "Espera a que un subproceso termine y recopila su salida. Lee stdout y stderr completos en goroutines para evitar deadlocks en pipes. Si timeoutSec > 0 y el proceso no termina en ese tiempo, llama a ProcessKill y marca el resultado con Killed=true. Retorna el exit code, salida completa y duracion total."
|
||||
tags: [process, subprocess, wait, timeout, exec, infra]
|
||||
uses_functions: [process_kill_go_infra]
|
||||
uses_types: [process_handle_go_infra, process_result_go_infra]
|
||||
returns: [process_result_go_infra]
|
||||
returns_optional: false
|
||||
error_type: "error_go_core"
|
||||
imports: [fmt, io, time]
|
||||
params:
|
||||
- name: handle
|
||||
desc: "handle del proceso lanzado por ProcessSpawn"
|
||||
- name: timeoutSec
|
||||
desc: "segundos maximos de espera; 0 o negativo espera indefinidamente"
|
||||
output: "resultado con exit code, stdout, stderr, duracion en ms y flag de killed"
|
||||
tested: true
|
||||
tests:
|
||||
- "spawn and wait echo"
|
||||
- "spawn with timeout kills"
|
||||
- "spawn with env"
|
||||
- "spawn script"
|
||||
- "spawn with working dir"
|
||||
test_file_path: "functions/infra/process_spawn_test.go"
|
||||
file_path: "functions/infra/process_wait.go"
|
||||
---
|
||||
|
||||
## Ejemplo
|
||||
|
||||
```go
|
||||
h, err := ProcessSpawn("sleep 60", "", nil, "")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
res, err := ProcessWait(h, 5) // timeout de 5 segundos
|
||||
if res.Killed {
|
||||
fmt.Println("proceso terminado por timeout")
|
||||
}
|
||||
```
|
||||
|
||||
## Notas
|
||||
|
||||
Funcion impura: bloquea esperando I/O y posiblemente llama a ProcessKill. Lee stdout y stderr en goroutines separadas antes de llamar a cmd.Wait() para evitar el deadlock clasico donde cmd.Wait() bloquea porque los pipes estan llenos y nadie los lee. El exit code -1 indica que ProcessState no estaba disponible (proceso matado antes de registrar estado).
|
||||
BIN
Binary file not shown.
@@ -234,3 +234,172 @@ repos:
|
||||
- id: wails_bind_crud_go_infra
|
||||
source_file: ""
|
||||
date: 2026-04-01
|
||||
|
||||
- repo: https://gitea-dgg044oo04woo4ggcsws4gk0.organic-machine.com/egutierrez/DevLauncher.git
|
||||
license: MIT
|
||||
cloned_dir: DevLauncher
|
||||
extracted:
|
||||
# Phase 1: Go Pure — TUI (5)
|
||||
- id: apply_gradient_go_tui
|
||||
source_file: launcher/core/gradient.go
|
||||
date: 2026-04-08
|
||||
- id: strip_ansi_go_tui
|
||||
source_file: launcher/core/commands.go
|
||||
date: 2026-04-08
|
||||
- id: normalize_terminal_output_go_tui
|
||||
source_file: launcher/core/commands.go
|
||||
date: 2026-04-08
|
||||
- id: draw_box_go_tui
|
||||
source_file: launcher/ui/styles.go
|
||||
date: 2026-04-08
|
||||
- id: draw_separator_go_tui
|
||||
source_file: launcher/ui/styles.go
|
||||
date: 2026-04-08
|
||||
# Phase 2: Go Pure — Core (5)
|
||||
- id: longest_common_prefix_go_core
|
||||
source_file: launcher/core/commands.go
|
||||
date: 2026-04-08
|
||||
- id: split_command_and_arg_go_core
|
||||
source_file: launcher/core/commands.go
|
||||
date: 2026-04-08
|
||||
- id: compare_versions_go_core
|
||||
source_file: installer/core/version.go
|
||||
date: 2026-04-08
|
||||
- id: parse_version_go_core
|
||||
source_file: installer/core/version.go
|
||||
date: 2026-04-08
|
||||
- id: rel_or_full_go_core
|
||||
source_file: launcher/core/script_query.go
|
||||
date: 2026-04-08
|
||||
# Phase 3: Go Impure (3)
|
||||
- id: load_ascii_art_go_tui
|
||||
source_file: launcher/middleware/assets.go
|
||||
date: 2026-04-08
|
||||
- id: read_dir_autocomplete_go_tui
|
||||
source_file: launcher/middleware/command_fs.go
|
||||
date: 2026-04-08
|
||||
- id: extract_script_description_go_shell
|
||||
source_file: launcher/middleware/reader.go
|
||||
date: 2026-04-08
|
||||
# Phase 4: Bash Library (6)
|
||||
- id: bash_colors_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
- id: bash_log_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
- id: bash_check_deps_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
- id: bash_confirm_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
- id: bash_safe_run_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
- id: bash_handle_error_bash_shell
|
||||
source_file: scripts/lib/common.sh
|
||||
date: 2026-04-08
|
||||
# Phase 5: Bash Cybersecurity (12)
|
||||
- id: analyze_dns_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/redes/analisis_dns.sh
|
||||
date: 2026-04-08
|
||||
- id: list_active_connections_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/redes/conexiones_activas.sh
|
||||
date: 2026-04-08
|
||||
- id: geolocate_ip_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/redes/geoip.sh
|
||||
date: 2026-04-08
|
||||
- id: audit_ssh_config_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/sistema/auditar_ssh.sh
|
||||
date: 2026-04-08
|
||||
- id: check_firewall_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/sistema/firewall_status.sh
|
||||
date: 2026-04-08
|
||||
- id: detect_suspicious_users_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/sistema/usuarios_sospechosos.sh
|
||||
date: 2026-04-08
|
||||
- id: encrypt_file_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/utilidades/cifrar_archivo.sh
|
||||
date: 2026-04-08
|
||||
- id: generate_password_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/utilidades/generar_password.sh
|
||||
date: 2026-04-08
|
||||
- id: verify_file_hash_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/utilidades/verificar_hash.sh
|
||||
date: 2026-04-08
|
||||
- id: audit_http_headers_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/web/cabeceras_http.sh
|
||||
date: 2026-04-08
|
||||
- id: inspect_ssl_cert_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/web/ssl_cert_info.sh
|
||||
date: 2026-04-08
|
||||
- id: enumerate_subdomains_bash_cybersecurity
|
||||
source_file: scripts/linux/ciberseguridad/web/subdominios.sh
|
||||
date: 2026-04-08
|
||||
# Phase 6: Git Utils (4)
|
||||
- id: git_repo_status_bash_shell
|
||||
source_file: scripts/linux/git_utils/estado_repo.sh
|
||||
date: 2026-04-08
|
||||
- id: git_clean_branches_bash_shell
|
||||
source_file: scripts/linux/git_utils/limpiar_ramas.sh
|
||||
date: 2026-04-08
|
||||
- id: git_push_all_remotes_bash_shell
|
||||
source_file: scripts/linux/git_utils/push_todos_remotes.sh
|
||||
date: 2026-04-08
|
||||
- id: git_log_visual_bash_shell
|
||||
source_file: scripts/linux/git_utils/historial_commits.sh
|
||||
date: 2026-04-08
|
||||
# Phase 7: Infra — System (3)
|
||||
- id: analyze_disk_space_bash_infra
|
||||
source_file: scripts/linux/gestion_linux/espacio_disponible.sh
|
||||
date: 2026-04-08
|
||||
- id: list_listening_ports_bash_infra
|
||||
source_file: scripts/linux/gestion_linux/puertos_activos.sh
|
||||
date: 2026-04-08
|
||||
- id: detect_wsl_bash_infra
|
||||
source_file: scripts/linux/gestion_linux/wsl_host.sh
|
||||
date: 2026-04-08
|
||||
# Phase 7: Infra — Installers (7)
|
||||
- id: install_go_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_go.sh
|
||||
date: 2026-04-08
|
||||
- id: install_nodejs_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_nodejs.sh
|
||||
date: 2026-04-08
|
||||
- id: install_pnpm_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_pnpm.sh
|
||||
date: 2026-04-08
|
||||
- id: install_python312_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_python312.sh
|
||||
date: 2026-04-08
|
||||
- id: install_uv_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_uv.sh
|
||||
date: 2026-04-08
|
||||
- id: install_volta_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_volta.sh
|
||||
date: 2026-04-08
|
||||
- id: install_wails_bash_infra
|
||||
source_file: scripts/linux/instaladores/instalar_wails.sh
|
||||
date: 2026-04-08
|
||||
# Phase 8: Shell Utils + Init (4)
|
||||
- id: convert_text_case_bash_shell
|
||||
source_file: scripts/linux/conversores/conversor_texto.sh
|
||||
date: 2026-04-08
|
||||
- id: create_project_structure_bash_shell
|
||||
source_file: scripts/linux/inicializar_repos/functional_structure.sh
|
||||
date: 2026-04-08
|
||||
- id: init_go_module_bash_pipelines
|
||||
source_file: scripts/linux/inicializar_repos/go/init_go_module.sh
|
||||
date: 2026-04-08
|
||||
- id: init_go_project_bash_pipelines
|
||||
source_file: scripts/linux/inicializar_repos/go/init_go_proyect.sh
|
||||
date: 2026-04-08
|
||||
|
||||
- repo: https://github.com/daguflow/dagu
|
||||
license: GPL-3.0
|
||||
cloned_dir: dagu
|
||||
analyzed: true
|
||||
extracted: []
|
||||
# GPL-3.0: no code extracted. YAML format studied for compatibility in issue 0007 (dag_engine).
|
||||
# Our implementation is written from scratch with no dagu code copied.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: dag_continue_on
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagContinueOn struct {
|
||||
Failure bool
|
||||
Skipped bool
|
||||
}
|
||||
description: "Politica de continuacion de un step DAG ante fallos o saltos. Cuando Failure=true el DAG no se detiene si el step falla. Cuando Skipped=true tampoco se detiene si el step es saltado."
|
||||
tags: [dag, workflow, policy, error-handling]
|
||||
uses_types: []
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Embebido en DagStep. Corresponde al campo `continue_on` del YAML de Dagu.
|
||||
@@ -0,0 +1,31 @@
|
||||
---
|
||||
name: dag_definition
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagDefinition struct {
|
||||
Name string
|
||||
Description string
|
||||
Group string
|
||||
Type string
|
||||
WorkingDir string
|
||||
Shell string
|
||||
Env map[string]string
|
||||
Schedule []string
|
||||
Steps []DagStep
|
||||
HandlerOn DagHandlers
|
||||
Tags []string
|
||||
TimeoutSec int
|
||||
FilePath string
|
||||
}
|
||||
description: "Definicion completa de un workflow DAG parseada desde YAML compatible con Dagu. Contiene steps, handlers de ciclo de vida, variables de entorno, schedule y metadatos del flujo."
|
||||
tags: [dag, workflow, yaml, dagu, definition]
|
||||
uses_types: [dag_step_go_core, dag_handlers_go_core]
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Type puede ser "graph" (ejecucion paralela por niveles topologicos) o vacio (cadena secuencial). Schedule es siempre una lista de strings (normalizado desde string o lista en el YAML). FilePath se rellena opcionalmente por el caller para saber el origen del archivo.
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: dag_handlers
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagHandlers struct {
|
||||
Init []DagStep
|
||||
Success []DagStep
|
||||
Failure []DagStep
|
||||
Exit []DagStep
|
||||
}
|
||||
description: "Handlers de ciclo de vida de un DAG. Cada campo contiene steps que se ejecutan en el evento correspondiente: inicializacion, exito, fallo o salida (siempre). Corresponde a handler_on o handlers en el YAML de Dagu."
|
||||
tags: [dag, workflow, handlers, lifecycle]
|
||||
uses_types: [dag_step_go_core]
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Embebido en DagDefinition. Los campos handler_on y handlers son aliases en el YAML de Dagu — ambos se normalizan a este tipo. Cada handler puede ser un step unico o una lista de steps en el YAML.
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
name: dag_retry_policy
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagRetryPolicy struct {
|
||||
Limit int
|
||||
IntervalSec int
|
||||
}
|
||||
description: "Politica de reintentos automaticos para un step DAG. Limit es el numero maximo de reintentos e IntervalSec es el tiempo de espera entre intentos en segundos."
|
||||
tags: [dag, workflow, retry, policy]
|
||||
uses_types: []
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Embebido en DagStep. Corresponde al campo `retry_policy` del YAML de Dagu. Limit=0 significa sin reintentos.
|
||||
@@ -0,0 +1,33 @@
|
||||
---
|
||||
name: dag_step
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagStep struct {
|
||||
Name string
|
||||
ID string
|
||||
Description string
|
||||
Command string
|
||||
Script string
|
||||
Args []string
|
||||
Shell string
|
||||
Dir string
|
||||
Depends []string
|
||||
Env map[string]string
|
||||
ContinueOn DagContinueOn
|
||||
RetryPolicy DagRetryPolicy
|
||||
TimeoutSec int
|
||||
Output string
|
||||
Tags []string
|
||||
}
|
||||
description: "Un paso individual en un workflow DAG con command/script, dependencias y configuracion de reintentos. Soporta variables de entorno por step, directorio de trabajo propio y politica continue_on."
|
||||
tags: [dag, workflow, step, yaml, dagu]
|
||||
uses_types: [dag_continue_on_go_core, dag_retry_policy_go_core]
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. El campo ID es opcional y se usa como referencia en otros steps (ej: `${id.stdout}`). Si Name e ID estan ambos vacios, el step es invalido. Env del step se mergea sobre el Env del DAG padre durante la resolucion.
|
||||
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: dag_validation_result
|
||||
lang: go
|
||||
domain: core
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagValidationResult struct {
|
||||
Valid bool
|
||||
Errors []string
|
||||
Warnings []string
|
||||
Levels [][]string
|
||||
}
|
||||
description: "Resultado de validar un DagDefinition. Contiene errores estructurales (ciclos, depends invalidos, nombres duplicados), warnings (command+script simultaneos) y los niveles topologicos calculados si el DAG es valido."
|
||||
tags: [dag, validation, workflow, result]
|
||||
uses_types: []
|
||||
file_path: "functions/core/dag_definition.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Levels solo se rellena cuando Valid=true. Cada sub-slice de Levels contiene los nombres/IDs de steps que pueden ejecutarse en paralelo. El orden de Levels refleja el orden topologico del grafo de dependencias.
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
name: DagRun
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagRun struct {
|
||||
ID string
|
||||
DagName string
|
||||
DagPath string
|
||||
Status string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Trigger string
|
||||
Error string
|
||||
}
|
||||
description: "Representa una ejecucion de un workflow DAG. Almacenado en SQLite con estado, timestamps y trigger."
|
||||
tags: [dag, execution, run, workflow]
|
||||
uses_types: []
|
||||
file_path: "functions/infra/dag_run.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Status puede ser: pending, running, success, failed, cancelled.
|
||||
Trigger puede ser: manual, cron, api.
|
||||
@@ -0,0 +1,29 @@
|
||||
---
|
||||
name: DagStepResult
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type DagStepResult struct {
|
||||
ID string
|
||||
RunID string
|
||||
StepName string
|
||||
Status string
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
DurationMs int64
|
||||
Error string
|
||||
}
|
||||
description: "Resultado de la ejecucion de un step individual dentro de un DagRun. Captura exit code, stdout, stderr y duracion."
|
||||
tags: [dag, execution, step, result]
|
||||
uses_types: [DagRun_go_infra]
|
||||
file_path: "functions/infra/dag_run.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Status puede ser: pending, running, success, failed, skipped.
|
||||
@@ -0,0 +1,24 @@
|
||||
---
|
||||
name: process_handle
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type ProcessHandle struct {
|
||||
Cmd *exec.Cmd
|
||||
Pid int
|
||||
StartTime time.Time
|
||||
Dir string
|
||||
stdout io.ReadCloser
|
||||
stderr io.ReadCloser
|
||||
}
|
||||
description: "Handle de un subproceso en ejecucion. Contiene el comando, PID, tiempo de inicio, directorio de trabajo y los pipes de stdout/stderr (privados, leidos internamente por ProcessWait)."
|
||||
tags: [process, subprocess, handle, infra, exec]
|
||||
uses_types: []
|
||||
file_path: "functions/infra/process_handle.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto. Los campos stdout y stderr son privados para evitar lecturas concurrentes externas — ProcessWait los consume internamente. Cmd.SysProcAttr.Setpgid=true garantiza que ProcessKill puede matar el process group completo usando -Pid.
|
||||
@@ -0,0 +1,23 @@
|
||||
---
|
||||
name: process_result
|
||||
lang: go
|
||||
domain: infra
|
||||
version: "1.0.0"
|
||||
algebraic: product
|
||||
definition: |
|
||||
type ProcessResult struct {
|
||||
ExitCode int
|
||||
Stdout string
|
||||
Stderr string
|
||||
DurationMs int64
|
||||
Killed bool
|
||||
}
|
||||
description: "Resultado de un subproceso completado. Contiene codigo de salida, salida estandar y de error, duracion en milisegundos, y un flag que indica si fue terminado por timeout."
|
||||
tags: [process, subprocess, result, exit, infra, exec]
|
||||
uses_types: []
|
||||
file_path: "functions/infra/process_handle.go"
|
||||
---
|
||||
|
||||
## Notas
|
||||
|
||||
Tipo producto — todos los campos siempre presentes. Killed=true indica que ProcessWait agoto el timeout y llamo a ProcessKill; en ese caso ExitCode suele ser -1 o el codigo de SIGKILL segun el OS. DurationMs incluye el tiempo total desde ProcessSpawn hasta que Wait() retorno.
|
||||
Reference in New Issue
Block a user