From eeff744dfcf1988e3d518f9ba2e8b92e3a9a791a Mon Sep 17 00:00:00 2001 From: Egutierrez Date: Thu, 7 May 2026 02:09:33 +0200 Subject: [PATCH] feat: dagu backup DAG + pre-commit drift hook + sync 6 apps Priority 1: Daily backup automation via Dagu DAG (~/dagu/dags/fn_backup.yaml, schedule "0 3 * * *"). Backs up registry.db, each operations.db, and vaults via rsync --link-dest. Fixes set -e arithmetic bugs in rotate_backups.sh and backup_all.sh ((var++) returns 1 when var=0). Fixes && chain set -e bug in vault rotation. Priority 2: Pre-commit hook v2 chains scan_secrets + uses_functions audit. New function git_hook_audit_app_drift_bash_infra blocks commits that touch app code when that app has uses_functions drift. Allows corrective app.md-only edits. Installed on fn_registry + 32 sub-repos. Priority 3: Synced uses_functions in 6 sub-repo apps (commits in their own repos): dag_engine, script_navegador, deploy_server, docker_tui, auto_metabase, metabase_registry. Drift went from 7/12 to 4/12 apps. Remaining drift = audit heuristic limitations (Python nested imports, Go symbol name detection). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../infra/git_hook_audit_app_drift.md | 45 ++++++ .../infra/git_hook_audit_app_drift.sh | 153 ++++++++++++++++++ .../infra/pre_commit_hook_install.sh | 37 +++-- bash/functions/infra/rotate_backups.sh | 8 +- bash/functions/pipelines/backup_all.sh | 27 ++-- 5 files changed, 246 insertions(+), 24 deletions(-) create mode 100644 bash/functions/infra/git_hook_audit_app_drift.md create mode 100755 bash/functions/infra/git_hook_audit_app_drift.sh diff --git a/bash/functions/infra/git_hook_audit_app_drift.md b/bash/functions/infra/git_hook_audit_app_drift.md new file mode 100644 index 00000000..9b41d42f --- /dev/null +++ b/bash/functions/infra/git_hook_audit_app_drift.md @@ -0,0 +1,45 @@ +--- +name: git_hook_audit_app_drift +kind: function +lang: bash +domain: infra +version: 1.0.0 +purity: impure +signature: "git_hook_audit_app_drift " +description: "Pre-commit guard: bloquea commit si los archivos staged tocan una app cuyo app.md tiene drift de uses_functions. Permite ediciones a app.md (correcciones)." +tags: [git, hook, precommit, registry-first, audit] +uses_functions: + - audit_uses_functions_go_infra +uses_types: [] +returns: [] +returns_optional: false +error_type: "error_go_core" +imports: [] +example: | + # Manual check + bash bash/functions/infra/git_hook_audit_app_drift.sh /home/lucas/fn_registry/apps/kanban + + # Used by pre_commit_hook_install_bash_infra (v2 hook chain) +file_path: bash/functions/infra/git_hook_audit_app_drift.sh +tested: false +params: + - name: repo_dir + desc: "Ruta absoluta o relativa al repo git a verificar (debe contener .git/)." +output: "stdout: vacio si OK. stderr: lineas '[hook] BLOCK: ...' si bloquea. Exit 0 = allow, 1 = block, 2 = error de config." +notes: | + Resuelve fn_registry root via FN_REGISTRY_ROOT, presencia de registry.db en + el repo, o ancestros (..\, ../..). Si no encuentra registry.db, salta el + check (no bloquea — el repo puede no estar fn_registry-aware). + + Llama `./fn doctor uses-functions --json` y filtra por DirPath registry-relativo. + Solo bloquea si la app del archivo staged aparece en el reporte CON drift + (Missing o Unused no vacios). + + Permite commits que tocan SOLO `app.md` (caso corrective: el dev esta + arreglando el frontmatter). Se reconoce por basename == 'app.md'. + + Para saltar: `git commit --no-verify` (debe ser raro y consciente). +documentation: | + Compone con `pre_commit_hook_install_bash_infra` para encadenar checks: + scan_secrets primero, luego este. Ambos fallan en exit != 0. +--- diff --git a/bash/functions/infra/git_hook_audit_app_drift.sh b/bash/functions/infra/git_hook_audit_app_drift.sh new file mode 100755 index 00000000..18dba3de --- /dev/null +++ b/bash/functions/infra/git_hook_audit_app_drift.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# git_hook_audit_app_drift - Pre-commit guard: block commit if staged code +# changes target an app with uses_functions drift in its app.md. +# +# Skips: +# - Commits where ALL staged files are app.md (corrective edits) +# - Apps without drift +# - Repos that aren't fn_registry-aware (no FN_REGISTRY_ROOT and no registry.db at top) +# +# Returns 0 (allow), 1 (block), 2 (config error) + +set -euo pipefail + +git_hook_audit_app_drift() { + local repo_dir="${1:-}" + if [[ -z "$repo_dir" ]]; then + echo "ERROR: repo_dir required" >&2 + return 2 + fi + if [[ ! -d "$repo_dir/.git" ]]; then + echo "ERROR: $repo_dir is not a git repo" >&2 + return 2 + fi + + # Find fn_registry root + local registry_root="${FN_REGISTRY_ROOT:-}" + if [[ -z "$registry_root" ]]; then + if [[ -f "$repo_dir/registry.db" ]]; then + registry_root="$repo_dir" + elif [[ -f "$repo_dir/../../registry.db" ]]; then + registry_root="$(cd "$repo_dir/../.." && pwd)" + elif [[ -f "$repo_dir/../../../registry.db" ]]; then + registry_root="$(cd "$repo_dir/../../.." && pwd)" + fi + fi + if [[ -z "$registry_root" || ! -f "$registry_root/registry.db" ]]; then + echo "[hook] FN_REGISTRY_ROOT not resolvable; skipping uses_functions check" >&2 + return 0 + fi + + local fn_bin="$registry_root/fn" + if [[ ! -x "$fn_bin" ]]; then + echo "[hook] $fn_bin not built; skipping uses_functions check" >&2 + return 0 + fi + + # Get staged files relative to repo root + local staged + staged=$(git -C "$repo_dir" diff --cached --name-only --diff-filter=ACMR) + if [[ -z "$staged" ]]; then + return 0 + fi + + # Detect which apps have changed code (any file other than app.md) + # An app dir contains an app.md. Find ancestor with app.md for each staged file. + local -A touched_apps=() + local rel + while IFS= read -r rel; do + # Skip pure app.md edits — those are corrective + local base + base=$(basename "$rel") + if [[ "$base" == "app.md" ]]; then + continue + fi + # Walk up looking for app.md (including repo root) + local dir + dir=$(dirname "$rel") + # First check repo root itself + if [[ -f "$repo_dir/app.md" ]]; then + touched_apps["."]=1 + continue + fi + while [[ "$dir" != "." && "$dir" != "/" ]]; do + if [[ -f "$repo_dir/$dir/app.md" ]]; then + touched_apps["$dir"]=1 + break + fi + dir=$(dirname "$dir") + done + done <<< "$staged" + + if [[ ${#touched_apps[@]} -eq 0 ]]; then + return 0 + fi + + # Get drift report from fn doctor (run from registry root so registry.db resolves) + local drift_json + drift_json=$(cd "$registry_root" && "$fn_bin" doctor uses-functions --json 2>/dev/null) || { + echo "[hook] fn doctor failed; allowing commit" >&2 + return 0 + } + + # Determine repo's location relative to registry_root, since touched_apps + # are repo-relative and audits are registry-relative. + local repo_abs + repo_abs=$(cd "$repo_dir" && pwd) + local registry_abs + registry_abs=$(cd "$registry_root" && pwd) + local repo_rel="${repo_abs#"$registry_abs/"}" + if [[ "$repo_rel" == "$repo_abs" ]]; then + repo_rel="" + fi + + # For each touched app, check if it appears in drift report. + # Drift JSON is a list of {AppID, DirPath, Lang, Missing, Unused}. + local blocking=0 + local app + for app in "${!touched_apps[@]}"; do + local dir_path_full + if [[ "$app" == "." ]]; then + dir_path_full="$repo_rel" + elif [[ -n "$repo_rel" ]]; then + dir_path_full="$repo_rel/$app" + else + dir_path_full="$app" + fi + + # Match by DirPath (registry-relative path) + local hit + hit=$(printf '%s' "$drift_json" | python3 -c " +import json, sys +data = json.load(sys.stdin) +target = '$dir_path_full'.rstrip('/') +for a in data or []: + dp = (a.get('DirPath') or '').rstrip('/') + if dp == target: + miss = a.get('Missing') or [] + unused = a.get('Unused') or [] + if miss or unused: + print(f\"{a.get('AppID','?')}|missing={','.join(miss[:5])}|unused={','.join(unused[:5])}\") + break +" 2>/dev/null) || hit="" + + if [[ -n "$hit" ]]; then + echo "[hook] BLOCK: app at $dir_path_full has uses_functions drift:" >&2 + echo " $hit" >&2 + blocking=1 + fi + done + + if [[ $blocking -eq 1 ]]; then + echo "" >&2 + echo "Fix app.md uses_functions, run \`./fn index\`, then retry commit." >&2 + echo "To bypass (not recommended): git commit --no-verify" >&2 + return 1 + fi + + return 0 +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + git_hook_audit_app_drift "$@" +fi diff --git a/bash/functions/infra/pre_commit_hook_install.sh b/bash/functions/infra/pre_commit_hook_install.sh index c9b4bb7e..5d5acd6a 100644 --- a/bash/functions/infra/pre_commit_hook_install.sh +++ b/bash/functions/infra/pre_commit_hook_install.sh @@ -15,7 +15,7 @@ pre_commit_hook_install() { local hooks_dir="$repo_dir/.git/hooks" local hook_path="$hooks_dir/pre-commit" - local marker="# fn_registry-pre-commit-v1" + local marker="# fn_registry-pre-commit-v2" if [[ ! -d "$hooks_dir" ]]; then echo "[pre_commit_hook_install] ERROR: '$repo_dir' no es un repo git valido (falta .git/hooks)" >&2 @@ -23,7 +23,8 @@ pre_commit_hook_install() { fi if [[ -f "$hook_path" ]]; then - if grep -qF "$marker" "$hook_path"; then + # Detect either v1 or v2 marker as "ours" + if grep -qE "fn_registry-pre-commit-v[12]" "$hook_path"; then if [[ $force -eq 0 ]]; then echo "SKIP $hook_path (already installed)" return 0 @@ -42,28 +43,46 @@ pre_commit_hook_install() { cat > "$hook_path" <<'HOOK' #!/usr/bin/env bash -# fn_registry-pre-commit-v1 +# fn_registry-pre-commit-v2 set -e -# Localizar fn_registry root: env var FN_REGISTRY_ROOT o asumir mismo repo si tiene registry.db en raiz +REPO_ROOT="$(git rev-parse --show-toplevel)" + +# Localizar fn_registry root REGISTRY_ROOT="${FN_REGISTRY_ROOT:-}" if [ -z "$REGISTRY_ROOT" ]; then - REPO_ROOT="$(git rev-parse --show-toplevel)" if [ -f "$REPO_ROOT/registry.db" ]; then REGISTRY_ROOT="$REPO_ROOT" + elif [ -f "$REPO_ROOT/../../registry.db" ]; then + REGISTRY_ROOT="$(cd "$REPO_ROOT/../.." && pwd)" + elif [ -f "$REPO_ROOT/../../../registry.db" ]; then + REGISTRY_ROOT="$(cd "$REPO_ROOT/../../.." && pwd)" fi fi -if [ -z "$REGISTRY_ROOT" ] || [ ! -f "$REGISTRY_ROOT/bash/functions/cybersecurity/scan_secrets_in_dirty.sh" ]; then - echo "[pre-commit] fn_registry no localizable; saltando scan de secrets" >&2 +if [ -z "$REGISTRY_ROOT" ] || [ ! -d "$REGISTRY_ROOT/bash/functions" ]; then + echo "[pre-commit] fn_registry no localizable; saltando checks" >&2 exit 0 fi -# Ejecutar scan en repo actual (cwd) -bash "$REGISTRY_ROOT/bash/functions/cybersecurity/scan_secrets_in_dirty.sh" "$(git rev-parse --show-toplevel)" +# Check 1: scan secrets +SECRETS_SH="$REGISTRY_ROOT/bash/functions/cybersecurity/scan_secrets_in_dirty.sh" +if [ -f "$SECRETS_SH" ]; then + bash "$SECRETS_SH" "$REPO_ROOT" +fi + +# Check 2: app uses_functions drift (only blocks if a touched app has drift) +DRIFT_SH="$REGISTRY_ROOT/bash/functions/infra/git_hook_audit_app_drift.sh" +if [ -f "$DRIFT_SH" ]; then + FN_REGISTRY_ROOT="$REGISTRY_ROOT" bash "$DRIFT_SH" "$REPO_ROOT" +fi HOOK chmod +x "$hook_path" echo "INSTALLED $hook_path" return 0 } + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + pre_commit_hook_install "$@" +fi diff --git a/bash/functions/infra/rotate_backups.sh b/bash/functions/infra/rotate_backups.sh index 0ea4f8a6..c9b4ed3f 100644 --- a/bash/functions/infra/rotate_backups.sh +++ b/bash/functions/infra/rotate_backups.sh @@ -23,7 +23,7 @@ rotate_backups() { esac ;; esac - ((i++)) + i=$((i + 1)) done # Validar numericos @@ -98,9 +98,9 @@ rotate_backups() { # Contar slots ocupados para el reporte local cnt_daily=0 cnt_weekly=0 cnt_monthly=0 - for (( j = 0; j < daily; j++ )); do [[ -e "$dir/daily.$j" ]] && ((cnt_daily++)); done - for (( j = 0; j < weekly; j++ )); do [[ -e "$dir/weekly.$j" ]] && ((cnt_weekly++)); done - for (( j = 0; j < monthly; j++ )); do [[ -e "$dir/monthly.$j" ]] && ((cnt_monthly++)); done + for (( j = 0; j < daily; j++ )); do [[ -e "$dir/daily.$j" ]] && cnt_daily=$((cnt_daily + 1)); done + for (( j = 0; j < weekly; j++ )); do [[ -e "$dir/weekly.$j" ]] && cnt_weekly=$((cnt_weekly + 1)); done + for (( j = 0; j < monthly; j++ )); do [[ -e "$dir/monthly.$j" ]] && cnt_monthly=$((cnt_monthly + 1)); done echo "ROTATED daily=$cnt_daily weekly=$cnt_weekly monthly=$cnt_monthly dir=$dir" } diff --git a/bash/functions/pipelines/backup_all.sh b/bash/functions/pipelines/backup_all.sh index 52420232..ad6b8db6 100644 --- a/bash/functions/pipelines/backup_all.sh +++ b/bash/functions/pipelines/backup_all.sh @@ -65,13 +65,13 @@ backup_all() { app_name="$(basename "$app_dir")" local snap_ops="/tmp/ops-snap-$$-${app_name}.db" if backup_sqlite_db "$ops_db" "$snap_ops"; then - rotate_backups "$backup_root/operations/$app_name" "$snap_ops" 7 4 12 || ((partial_errors++)) + rotate_backups "$backup_root/operations/$app_name" "$snap_ops" 7 4 12 || partial_errors=$((partial_errors + 1)) rm -f "$snap_ops" - ((ops_count++)) + ops_count=$((ops_count + 1)) else echo "WARN: Fallo snapshot de $ops_db — skipped" >&2 rm -f "$snap_ops" - ((partial_errors++)) + partial_errors=$((partial_errors + 1)) fi done < <(find "$registry_root/apps" "$registry_root/projects" -name "operations.db" -maxdepth 4 2>/dev/null || true) @@ -86,11 +86,16 @@ backup_all() { current_name="${BASH_REMATCH[1]}" elif [[ "$line" =~ ^[[:space:]]*path:[[:space:]]*(.+)$ && -n "$current_name" ]]; then local vault_path="${BASH_REMATCH[1]}" + # Strip surrounding quotes ("..." o '...') + vault_path="${vault_path%\"}" + vault_path="${vault_path#\"}" + vault_path="${vault_path%\'}" + vault_path="${vault_path#\'}" # Expandir ~ si fuera necesario vault_path="${vault_path/#\~/$HOME}" if [[ ! -d "$vault_path" ]]; then echo "WARN: Vault '$current_name' path '$vault_path' no existe — skipped" >&2 - ((partial_errors++)) + partial_errors=$((partial_errors + 1)) current_name="" continue fi @@ -107,7 +112,7 @@ backup_all() { # Rotacion manual de directorios (7 daily, 4 weekly, 12 monthly) _rotate_vault_dirs "$vault_dest" 7 4 12 mv "$tmp_dest" "$vault_dest/daily.0" - ((vaults_count++)) + vaults_count=$((vaults_count + 1)) current_name="" fi done < "$vault_yaml" @@ -143,22 +148,22 @@ _rotate_vault_dirs() { if [[ "$month_day" == "01" && -d "$dir/weekly.0" ]]; then for ((i=monthly-1; i>=1; i--)); do - [[ -d "$dir/monthly.$((i-1))" ]] && mv "$dir/monthly.$((i-1))" "$dir/monthly.$i" + if [[ -d "$dir/monthly.$((i-1))" ]]; then mv "$dir/monthly.$((i-1))" "$dir/monthly.$i"; fi done - [[ -d "$dir/weekly.0" ]] && cp -al "$dir/weekly.0" "$dir/monthly.0" + if [[ -d "$dir/weekly.0" ]]; then cp -al "$dir/weekly.0" "$dir/monthly.0"; fi fi if [[ "$week_day" == "7" && -d "$dir/daily.0" ]]; then for ((i=weekly-1; i>=1; i--)); do - [[ -d "$dir/weekly.$((i-1))" ]] && mv "$dir/weekly.$((i-1))" "$dir/weekly.$i" + if [[ -d "$dir/weekly.$((i-1))" ]]; then mv "$dir/weekly.$((i-1))" "$dir/weekly.$i"; fi done - [[ -d "$dir/daily.0" ]] && cp -al "$dir/daily.0" "$dir/weekly.0" + if [[ -d "$dir/daily.0" ]]; then cp -al "$dir/daily.0" "$dir/weekly.0"; fi fi # Rotar daily - [[ -d "$dir/daily.$((daily-1))" ]] && rm -rf "$dir/daily.$((daily-1))" + if [[ -d "$dir/daily.$((daily-1))" ]]; then rm -rf "$dir/daily.$((daily-1))"; fi for ((i=daily-1; i>=1; i--)); do - [[ -d "$dir/daily.$((i-1))" ]] && mv "$dir/daily.$((i-1))" "$dir/daily.$i" + if [[ -d "$dir/daily.$((i-1))" ]]; then mv "$dir/daily.$((i-1))" "$dir/daily.$i"; fi done }