# functions

id: bq_list_datasets_py_infra
name: bq_list_datasets
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
signature: def bq_list_datasets(client: BQClient) -> list[dict]
tags: ["bigquery","gcp","dataset","list","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/datasets.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 14f5de1951d1f470428e6ae7648f2d8e1e92d9d31b9cfdc680a3a8f89663b9f5
description:
  Lista todos los datasets del proyecto de Google BigQuery. Usa client._client.list_datasets() del SDK oficial.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.datasets import bq_list_datasets

  client = bq_auth("my-project")
  datasets = bq_list_datasets(client)
  for ds in datasets:
      print(ds["dataset_id"], ds["full_id"])
  # analytics  my-project.analytics
  # raw_data   my-project.raw_data
  ```

notes:
  Retorna solo campos basicos (dataset_id, project, full_id). Para obtener detalles completos (location, description, labels) usar `bq_get_dataset` sobre cada item.
  El SDK retorna un iterador; esta funcion lo materializa en una lista completa.
  Si el proyecto no tiene datasets, retorna lista vacia `[]`.

code:
[cell repeats an earlier one in this file: 5525 characters]

params_schema:
  {"params":[{"name":"client","desc":"instancia autenticada de BQClient"}],"output":"lista de dicts con dataset_id, project y full_id por cada dataset del proyecto"}


id: bq_list_jobs_py_infra
name: bq_list_jobs
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","job","list","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/jobs.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: edd6fbfa4c18ec83a16b924cd201b2062d558dfe7ba84ad6bfadbf0b527ebd90
signature:
  def bq_list_jobs(client: BQClient, state_filter: str = '', max_results: int = 50, all_users: bool = False) -> list[dict]

description:
  Lista jobs del proyecto con filtro por estado (running, pending, done). Retorna una lista de dicts planos con metadatos de cada job.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.jobs import bq_list_jobs

  client = bq_auth("my-project")

  # Todos los jobs recientes
  jobs = bq_list_jobs(client, max_results=20)

  # Solo jobs en ejecucion
  running = bq_list_jobs(client, state_filter="running")
  for j in running:
      print(j["job_id"], j["state"], j["job_type"])

  # Jobs completados de todos los usuarios
  done = bq_list_jobs(client, state_filter="done", all_users=True, max_results=100)
  ```

notes:
  Los campos `created`, `started` y `ended` se serializan como strings ISO 8601 (o `None` si no estan disponibles).

  El campo `bytes_processed` solo esta disponible en query jobs; para otros tipos de job (load, export, copy) se retorna `None`.

  El campo `error` contiene el error_result como string si el job fallo, o `None` si no hay error.

  Para obtener el texto completo del SQL de un job concreto, usar `bq_get_job` que incluye el campo `query`.

code:
  """Gestion de jobs en Google BigQuery."""

  from .client import BQClient


  def bq_list_jobs(
      client: BQClient,
      state_filter: str = "",
      max_results: int = 50,
      all_users: bool = False,
  ) -> list[dict]:
      """Lista jobs del proyecto con filtro opcional por estado.

      Args:
          client: Cliente autenticado.
          state_filter: Filtro: "running", "pending", "done". Vacio = todos.
          max_results: Numero maximo de jobs a retornar.
          all_users: Si True, lista jobs de todos los usuarios (requiere permisos).

      Returns:
          Lista de dicts con: job_id, job_type, state, created, started, ended,
          user_email, bytes_processed, error.

      Example:
          >>> jobs = bq_list_jobs(client, state_filter="running")
          >>> for j in jobs:
          ...     print(j["job_id"], j["state"], j["job_type"])
      """
      kwargs = {"max_results": max_results, "all_users": all_users}
      if state_filter:
          kwargs["state_filter"] = state_filter
      return [_job_to_dict(job) for job in client._client.list_jobs(**kwargs)]


  def bq_get_job(client: BQClient, job_id: str) -> dict:
      """Obtiene detalles de un job por su ID.

      Args:
          client: Cliente autenticado.
          job_id: ID del job.

      Returns:
          Dict con: job_id, job_type, state, created, started, ended,
          user_email, bytes_processed, destination_table, query, error, errors.

      Raises:
          google.api_core.exceptions.NotFound: Si el job no existe.

      Example:
          >>> job = bq_get_job(client, "job_abc123")
          >>> print(job["state"], job["bytes_processed"])
          >>> if job["error"]:
          ...     print("Error:", job["error"])
      """
      job = client._client.get_job(job_id)
      result = _job_to_dict(job)
      # Agregar detalles extra disponibles en get pero no en list
      if hasattr(job, 'destination') and job.destination:
          result["destination_table"] = str(job.destination)
      if hasattr(job, 'query') and job.query:
          result["query"] = job.query
      if job.errors:
          result["errors"] = [dict(e) for e in job.errors]
      return result


  def bq_cancel_job(client: BQClient, job_id: str) -> dict:
      """Cancela un job en ejecucion.

      Args:
          client: Cliente autenticado.
          job_id: ID del job a cancelar.

      Returns:
          Dict con: job_id, state (tras cancelacion).

      Raises:
          google.api_core.exceptions.NotFound: Si el job no existe.

      Example:
          >>> result = bq_cancel_job(client, "job_abc123")
          >>> print(result["state"])  # "DONE" (cancelled)
      """
      job = client._client.cancel_job(job_id)
      return {"job_id": job.job_id, "state": job.state}


  def _job_to_dict(job) -> dict:
      """Convierte un objeto Job del SDK a dict plano."""
      result = {
          "job_id": job.job_id,
          "job_type": job.job_type,
          "state": job.state,
          "created": job.created.isoformat() if job.created else None,
          "started": job.started.isoformat() if job.started else None,
          "ended": job.ended.isoformat() if job.ended else None,
          "user_email": job.user_email,
          "error": str(job.error_result) if job.error_result else None,
      }
      # bytes_processed solo disponible en query jobs
      try:
          result["bytes_processed"] = job.total_bytes_processed
      except AttributeError:
          result["bytes_processed"] = None
      return result

params_schema:
  {"params":[{"name":"client","desc":"cliente autenticado de BigQuery obtenido con bq_auth"},{"name":"state_filter","desc":"filtro por estado del job: 'running', 'pending' o 'done'; vacio = todos los estados"},{"name":"max_results","desc":"numero maximo de jobs a retornar (por defecto 50)"},{"name":"all_users","desc":"si True lista jobs de todos los usuarios del proyecto; requiere permisos de administrador"}],"output":"lista de dicts con campos: job_id, job_type, state, created, started, ended, user_email, bytes_processed, error"}


id: bq_list_routines_py_infra
name: bq_list_routines
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
signature: def bq_list_routines(client: BQClient, dataset_id: str) -> list[dict]
description: Lista routines de un dataset incluyendo tipo y lenguaje.
tags: ["bigquery","gcp","routine","udf","list","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/routines.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: e02e9c7a8aa5320857ee6f5dc205a95407b0cbd93fbbfe963394139e47ad1da5
example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.routines import bq_list_routines

  client = bq_auth("my-project")
  routines = bq_list_routines(client, "analytics")

  for r in routines:
      print(r["routine_id"], r["routine_type"], r["language"])
  # double_value SCALAR_FUNCTION SQL
  # refresh_summary PROCEDURE SQL
  # parse_json SCALAR_FUNCTION JAVASCRIPT
  ```

notes:
  Retorna lista vacia si el dataset no tiene routines. Lanza `google.api_core.exceptions.NotFound` si el dataset no existe.

  El listado solo incluye metadata basica (routine_id, tipo, lenguaje). Para obtener el cuerpo y argumentos completos de una routine especifica, usar `client._client.get_routine(f"{project}.{dataset}.{routine_id}")`.

code:
  """Gestion de routines (UDFs y stored procedures) en Google BigQuery."""

  from .client import BQClient
  from google.cloud import bigquery


  def bq_create_routine(
      client: BQClient,
      dataset_id: str,
      routine_id: str,
      body: str,
      routine_type: str = "SCALAR_FUNCTION",
      language: str = "SQL",
      arguments: list[dict] | None = None,
      return_type: str = "",
      description: str = "",
  ) -> dict:
      """Crea una routine (UDF o stored procedure) en BigQuery.

      Args:
          client: Cliente autenticado.
          dataset_id: ID del dataset donde crear la routine.
          routine_id: ID/nombre de la routine.
          body: Cuerpo de la routine (SQL, JavaScript o Python).
          routine_type: Tipo: "SCALAR_FUNCTION", "TABLE_VALUED_FUNCTION", "PROCEDURE".
          language: Lenguaje: "SQL", "JAVASCRIPT", "PYTHON".
          arguments: Lista de argumentos, cada uno: {"name": "x", "data_type": "STRING"}.
          return_type: Tipo de retorno para funciones (ej: "STRING", "INT64"). No aplica a procedures.
          description: Descripcion opcional.

      Returns:
          Dict con: routine_id, dataset_id, project, routine_type, language, body, created, modified.

      Raises:
          google.api_core.exceptions.Conflict: Si la routine ya existe.

      Example:
          >>> bq_create_routine(client, "analytics", "double_value",
          ...     body="x * 2",
          ...     arguments=[{"name": "x", "data_type": "INT64"}],
          ...     return_type="INT64")
      """
      ref = bigquery.RoutineReference.from_string(
          f"{client.project_id}.{dataset_id}.{routine_id}"
      )
      routine = bigquery.Routine(ref)
      routine.type_ = routine_type
      routine.language = language
      routine.body = body
      if description:
          routine.description = description
      if arguments:
          routine.arguments = [
              bigquery.RoutineArgument(
                  name=arg["name"],
                  data_type=bigquery.StandardSqlDataType(
                      type_kind=getattr(
                          bigquery.StandardSqlTypeNames, arg["data_type"]
                      )
                  ),
              )
              for arg in arguments
          ]
      if return_type:
          routine.return_type = bigquery.StandardSqlDataType(
              type_kind=getattr(bigquery.StandardSqlTypeNames, return_type)
          )
      created = client._client.create_routine(routine)
      return _routine_to_dict(created)


  def bq_list_routines(client: BQClient, dataset_id: str) -> list[dict]:
      """Lista routines de un dataset.

      Args:
          client: Cliente autenticado.
          dataset_id: ID del dataset.

      Returns:
          Lista de dicts con: routine_id, dataset_id, project, routine_type, language.

      Example:
          >>> routines = bq_list_routines(client, "analytics")
          >>> for r in routines:
          ...     print(r["routine_id"], r["routine_type"], r["language"])
      """
      ref = f"{client.project_id}.{dataset_id}"
      return [
          {
              "routine_id": r.routine_id,
              "dataset_id": dataset_id,
              "project": client.project_id,
              "routine_type": r.type_,
              "language": r.language,
          }
          for r in client._client.list_routines(ref)
      ]


  def bq_delete_routine(
      client: BQClient,
      dataset_id: str,
      routine_id: str,
  ) -> None:
      """Elimina una routine.

      Args:
          client: Cliente autenticado.
          dataset_id: ID del dataset.
          routine_id: ID de la routine a eliminar.

      Raises:
          google.api_core.exceptions.NotFound: Si la routine no existe.

      Example:
          >>> bq_delete_routine(client, "analytics", "double_value")
      """
      ref = f"{client.project_id}.{dataset_id}.{routine_id}"
      client._client.delete_routine(ref)


  def _routine_to_dict(routine) -> dict:
      """Convierte un objeto Routine del SDK a dict plano."""
      return {
          "routine_id": routine.routine_id,
          "dataset_id": routine.dataset_id,
          "project": routine.project,
          "routine_type": routine.type_,
          "language": routine.language,
          "body": routine.body,
          "description": routine.description or "",
          "created": routine.created.isoformat() if routine.created else None,
          "modified": routine.modified.isoformat() if routine.modified else None,
      }

params_schema:
  {"params":[{"name":"client","desc":"instancia autenticada de BQClient"},{"name":"dataset_id","desc":"nombre del dataset cuyas routines se quieren listar"}],"output":"lista de dicts con routine_id, dataset_id, project, routine_type y language para cada routine del dataset"}


id: bq_list_tables_py_infra
name: bq_list_tables
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
signature: def bq_list_tables(client: BQClient, dataset_id: str) -> list[dict]
tags: ["bigquery","gcp","table","list","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/tables.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 838b843ab86012aebf67a343b23aa8b033b7d04c42089abc99a9ff6a75e927fb
description:
  Lista todas las tablas (y vistas) de un dataset BigQuery con informacion resumida. Usa client._client.list_tables() del SDK oficial.

example:
  ```python
  from bigquery import bq_auth, bq_list_tables

  client = bq_auth("mi-proyecto")

  tablas = bq_list_tables(client, "ventas_ds")
  for t in tablas:
      print(t["table_id"], t["type"])
      # transacciones TABLE
      # vista_mensual VIEW

  # Filtrar solo tablas fisicas
  solo_tablas = [t for t in tablas if t["type"] == "TABLE"]
  ```

notes:
  Retorna objetos resumidos (`ListTableItem`), no los metadatos completos. Para obtener schema, estadisticas y configuracion completa usar `bq_get_table`. El listado incluye tablas, vistas, vistas materializadas y tablas externas (EXTERNAL). El orden de retorno no esta garantizado.

code:
[cell repeats an earlier one in this file: 12315 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente autenticado BQClient obtenido con bq_auth"},{"name":"dataset_id","desc":"ID del dataset a listar"}],"output":"lista de dicts, uno por objeto del dataset. Cada dict contiene: table_id (nombre), full_id (project.dataset.table), type (TABLE, VIEW, MATERIALIZED_VIEW, EXTERNAL)"}


id: bq_load_from_file_py_infra
name: bq_load_from_file
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","load","file","google-cloud","python","etl"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/queries.py
created_at: 2026-04-07T09:56:45Z
updated_at: 2026-04-07T09:56:45Z
props: []
emits: []
variant: []
content_hash: 62dc9cf942b94d0c28c2ce33472e1386e87142391205c15364666636676499c2
signature:
  def bq_load_from_file(client: BQClient, file_path: str, dataset_id: str, table_id: str, source_format: str = 'CSV', write_disposition: str = 'WRITE_APPEND', autodetect: bool = True, skip_leading_rows: int = 0) -> dict

description:
  Carga datos desde un archivo local a una tabla BigQuery usando load_table_from_file del SDK. Equivalente a bq_load_from_gcs pero para disco local.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.queries import bq_load_from_file

  client = bq_auth("my-project")

  # Cargar CSV local con cabecera
  result = bq_load_from_file(
      client,
      "/tmp/export_users.csv",
      "my_dataset", "users",
      skip_leading_rows=1,
      write_disposition="WRITE_TRUNCATE",
  )
  print(f"Cargadas {result['rows_loaded']} filas — job: {result['job_id']}")

  # Cargar JSONL local
  result = bq_load_from_file(
      client,
      "/data/events.jsonl",
      "my_dataset", "events",
      source_format="NEWLINE_DELIMITED_JSON",
  )
  ```

notes:
  El archivo se abre en modo binario (`rb`) y se sube directamente al job de BigQuery.
  Para archivos muy grandes, preferir `bq_load_from_gcs` — subir primero a GCS y luego
  cargar desde ahi es mas eficiente y permite paralelismo.

  La funcion bloquea hasta que el job termina (`job.result()`). Los archivos Parquet y
  Avro no admiten `skip_leading_rows` — ese parametro solo aplica para CSV.

code:
[cell repeats an earlier one in this file: 14804 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente BQClient autenticado contra el proyecto GCP"},{"name":"file_path","desc":"ruta absoluta o relativa al archivo local a cargar"},{"name":"dataset_id","desc":"ID del dataset de destino en BigQuery"},{"name":"table_id","desc":"ID de la tabla de destino en BigQuery"},{"name":"source_format","desc":"formato del archivo fuente: CSV, NEWLINE_DELIMITED_JSON, AVRO, PARQUET, ORC"},{"name":"write_disposition","desc":"comportamiento si la tabla ya existe: WRITE_APPEND agrega, WRITE_TRUNCATE reemplaza, WRITE_EMPTY falla si hay datos"},{"name":"autodetect","desc":"si True, BigQuery infiere el schema automaticamente desde los datos"},{"name":"skip_leading_rows","desc":"numero de filas a ignorar al inicio del archivo (tipicamente 1 para saltar cabeceras CSV)"}],"output":"dict con {job_id: ID del LoadJob, rows_loaded: filas cargadas, status: DONE o FAILED}"}


id: bq_load_from_gcs_py_infra
name: bq_load_from_gcs
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","load","gcs","google-cloud","python","etl"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/queries.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 7a0ef734f51a22ae64a9e632e334b7298beb8442d6588636c61439d4e67ef3ed
signature:
  def bq_load_from_gcs(client: BQClient, uri: str | list[str], dataset_id: str, table_id: str, source_format: str = 'CSV', write_disposition: str = 'WRITE_APPEND', autodetect: bool = True, skip_leading_rows: int = 0) -> dict

description:
  Carga datos desde uno o varios URIs de Google Cloud Storage a una tabla BigQuery configurando un LoadJob. Espera la finalizacion del job.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.queries import bq_load_from_gcs

  client = bq_auth("my-project")

  # Cargar un archivo CSV con cabecera
  result = bq_load_from_gcs(
      client,
      "gs://my-bucket/data/users_2024.csv",
      "my_dataset", "users",
      skip_leading_rows=1,
  )
  print(f"Cargadas {result['rows_loaded']} filas — job: {result['job_id']}")

  # Cargar multiples archivos Parquet reemplazando la tabla
  result = bq_load_from_gcs(
      client,
      ["gs://my-bucket/export/part_001.parquet", "gs://my-bucket/export/part_002.parquet"],
      "my_dataset", "events",
      source_format="PARQUET",
      write_disposition="WRITE_TRUNCATE",
  )
  ```

notes:
  El job se ejecuta de forma asincrona en BigQuery; `job.result()` bloquea hasta completar.

  Los wildcards en el URI (`gs://bucket/prefix_*.csv`) son resueltos por GCS — BigQuery
  acepta la lista de archivos resultante como una sola carga atomica.

  `autodetect=True` es conveniente pero puede inferir tipos incorrectamente para columnas
  con valores nulos o mixtos. Para produccion, definir el schema explicitamente via
  `job_config.schema`.

code:
[cell repeats an earlier one in this file: 14804 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente BQClient autenticado contra el proyecto GCP"},{"name":"uri","desc":"URI de GCS de origen (gs://bucket/file.csv) o lista de URIs; soporta wildcards como gs://bucket/prefix_*.csv"},{"name":"dataset_id","desc":"ID del dataset de destino en BigQuery"},{"name":"table_id","desc":"ID de la tabla de destino en BigQuery"},{"name":"source_format","desc":"formato del archivo fuente: CSV, NEWLINE_DELIMITED_JSON, AVRO, PARQUET, ORC"},{"name":"write_disposition","desc":"comportamiento si la tabla ya existe: WRITE_APPEND agrega, WRITE_TRUNCATE reemplaza, WRITE_EMPTY falla si hay datos"},{"name":"autodetect","desc":"si True, BigQuery infiere el schema automaticamente desde los datos"},{"name":"skip_leading_rows","desc":"numero de filas a ignorar al inicio del archivo (tipicamente 1 para saltar cabeceras CSV)"}],"output":"dict con {job_id: ID del LoadJob, rows_loaded: filas cargadas, status: DONE o FAILED}"}


id: bq_preview_rows_py_infra
name: bq_preview_rows
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","table","preview","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/tables.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 0774694fcdd4158647410234c911531c203a98c0fb7428f63f804da40b7227bd
signature:
  def bq_preview_rows(client: BQClient, dataset_id: str, table_id: str, max_results: int = 10) -> dict

description:
  Obtiene una muestra de filas de una tabla BigQuery sin ejecutar query SQL, sin coste de procesamiento. Usa client._client.list_rows() del SDK oficial.

example:
  ```python
  from bigquery import bq_auth, bq_preview_rows

  client = bq_auth("mi-proyecto")

  preview = bq_preview_rows(client, "ventas_ds", "transacciones", max_results=5)

  print(preview["columns"])         # ["id", "fecha", "monto", "pais"]
  print(f"Total filas: {preview['total_rows']}")
  for row in preview["rows"]:
      print(row)                    # [1, datetime.date(2024, 1, 15), 99.5, "MX"]
  ```

notes:
  `list_rows()` usa la Storage Read API internamente y NO genera un job de query — por tanto no se contabiliza en el uso de bytes procesados. Ideal para inspeccionar rapidamente la estructura y contenido de una tabla. El orden de las filas retornadas no esta garantizado (depende del almacenamiento interno de BigQuery). Para muestras reproducibles o con filtros, usar una query SQL con `LIMIT`. `total_rows` refleja el conteo de la tabla en el momento de la llamada a `get_table()`, que puede tener un pequeno retraso respecto al dato real.

code:
[cell repeats an earlier one in this file: 12315 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente autenticado BQClient obtenido con bq_auth"},{"name":"dataset_id","desc":"ID del dataset que contiene la tabla"},{"name":"table_id","desc":"nombre (ID) de la tabla a previsualizar"},{"name":"max_results","desc":"numero maximo de filas a retornar (default: 10)"}],"output":"dict con: columns (lista de nombres de columnas), rows (lista de listas con valores), total_rows (int con total de filas en la tabla completa)"}


id: bq_query_py_infra
name: bq_query
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","query","sql","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/queries.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 37a3af3caa5aa5cf0ca25fcd2f64446261a98296da66e539be289b648d03411b
signature:
  def bq_query(client: BQClient, sql: str, params: list[dict] | None = None, dry_run: bool = False) -> dict

description:
  Ejecuta una query SQL en BigQuery con soporte para parametros tipados y modo dry-run para estimacion de costos.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.queries import bq_query

  client = bq_auth("my-project")

  # Query simple
  result = bq_query(client, "SELECT COUNT(*) as n FROM `my_project.dataset.table`")
  print(result["columns"])  # ["n"]
  print(result["rows"])     # [[12345]]

  # Con parametros
  result = bq_query(
      client,
      "SELECT * FROM `dataset.users` WHERE status = @s AND age > @a",
      params=[
          {"name": "s", "type": "STRING", "value": "active"},
          {"name": "a", "type": "INT64",  "value": 18},
      ],
  )

  # Dry-run para estimar costo
  est = bq_query(client, "SELECT * FROM `dataset.big_table`", dry_run=True)
  print(f"Procesaria {est['total_bytes_processed'] / 1e9:.2f} GB")
  ```

notes:
  Usa `bigquery.ScalarQueryParameter` para parametros — solo soporta escalares. Para arrays
  usar `bigquery.ArrayQueryParameter` directamente si se necesita.

  En dry_run=True el job se crea pero no se ejecuta; `job.result()` no se llama. BigQuery
  retorna la estimacion de bytes sin cargo.

  `cache_hit=True` indica que el resultado provino de la cache de BigQuery (sin costo).

code:
[cell repeats an earlier one in this file: 14804 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente BQClient autenticado contra el proyecto GCP"},{"name":"sql","desc":"query SQL a ejecutar; usar @nombre para referencias a parametros tipados"},{"name":"params","desc":"lista de parametros tipados, cada uno con {name, type, value}; tipos: STRING, INT64, FLOAT64, BOOL, DATE, TIMESTAMP"},{"name":"dry_run","desc":"si True, estima bytes procesados/facturados sin ejecutar la query"}],"output":"si dry_run=True: {total_bytes_processed, total_bytes_billed}; si False: {columns, rows, total_rows, bytes_processed, cache_hit}"}


id: bq_update_dataset_py_infra
name: bq_update_dataset
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","dataset","update","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/datasets.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 497bd7e440ea911168b1c34b385eed136bf8759f72c20b20660e9ef4888af92e
signature:
  def bq_update_dataset(client: BQClient, dataset_id: str, description: str | None = None, labels: dict[str, str] | None = None, default_table_expiration_ms: int | None = None) -> dict

description:
  Actualiza campos de un dataset de Google BigQuery. Solo modifica los campos pasados explicitamente (no-None). Usa client._client.update_dataset() del SDK oficial.

example:
  ```python
  from bigquery.client import bq_auth
  from bigquery.datasets import bq_update_dataset

  client = bq_auth("my-project")

  # Actualizar solo la descripcion
  ds = bq_update_dataset(client, "analytics", description="Data warehouse actualizado")

  # Actualizar labels
  ds = bq_update_dataset(client, "analytics", labels={"env": "prod", "version": "2"})

  # Eliminar expiracion de tablas
  ds = bq_update_dataset(client, "analytics", default_table_expiration_ms=0)
  print(ds["description"])
  ```

notes:
  Lanza `google.api_core.exceptions.NotFound` (404) si el dataset no existe.
  Si no se pasa ningun campo (todos None), retorna el dataset sin modificar (no llama a update_dataset).
  Para eliminar todos los labels pasar `labels={}`. Para eliminar la expiracion de tablas pasar `default_table_expiration_ms=0`.
  El SDK hace un GET interno antes del PATCH para obtener el estado actual; esta funcion replica ese patron explicitamente.

code:
[cell repeats an earlier one in this file: 5525 characters]

params_schema:
  {"params":[{"name":"client","desc":"instancia autenticada de BQClient"},{"name":"dataset_id","desc":"nombre del dataset a actualizar (sin prefijo de proyecto)"},{"name":"description","desc":"nueva descripcion del dataset; None = no modificar"},{"name":"labels","desc":"nuevos labels key-value; None = no modificar; dict vacio {} = eliminar todos los labels"},{"name":"default_table_expiration_ms","desc":"nueva expiracion por defecto de tablas en ms; None = no modificar; 0 = eliminar expiracion"}],"output":"dict con el dataset actualizado: dataset_id, project, full_id, location, description, labels, created, modified, default_table_expiration_ms"}


id: bq_update_table_py_infra
name: bq_update_table
kind: function
lang: py
domain: infra
version: 1.0.0
purity: impure
tags: ["bigquery","gcp","table","update","google-cloud","python"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["google-cloud-bigquery"]
tests: []
file_path: python/functions/bigquery/tables.py
created_at: 2026-04-07T09:56:37Z
updated_at: 2026-04-07T09:56:37Z
props: []
emits: []
variant: []
content_hash: 69a3b58d265a0e7420c353fe1b5d7d95e9c83c2afd3d8b7a01d48354f0f81b4d
signature:
  def bq_update_table(client: BQClient, dataset_id: str, table_id: str, schema: list[dict] | None = None, description: str | None = None, labels: dict | None = None) -> dict

description:
  Actualiza metadatos de una tabla BigQuery: schema (solo adicion de columnas), descripcion y etiquetas. Usa client._client.update_table() del SDK oficial.

example:
  ```python
  from bigquery import bq_auth, bq_update_table

  client = bq_auth("mi-proyecto")

  # Actualizar descripcion y etiquetas
  tabla = bq_update_table(
      client, "ventas_ds", "transacciones",
      description="Transacciones de ventas — actualizado",
      labels={"env": "prod", "team": "data", "version": "2"},
  )

  # Agregar columna nueva al schema existente
  tabla = bq_update_table(
      client, "ventas_ds", "transacciones",
      schema=[
          {"name": "id", "type": "INTEGER", "mode": "REQUIRED"},
          {"name": "fecha", "type": "DATE", "mode": "NULLABLE"},
          {"name": "monto", "type": "FLOAT", "mode": "NULLABLE"},
          {"name": "nueva_col", "type": "STRING", "mode": "NULLABLE"},  # nueva
      ],
  )
  ```

notes:
  BigQuery tiene restricciones estrictas sobre modificacion de schema: se pueden agregar columnas NULLABLE o REPEATED al final, pero NO se pueden eliminar columnas, renombrar columnas ni cambiar el tipo de una columna existente. Si se necesita ese tipo de cambio, la alternativa es crear una tabla nueva con `CREATE TABLE AS SELECT`. Los campos `None` no generan actualizacion — solo se envian al API los campos que cambian.

code:
[cell repeats an earlier one in this file: 12315 characters]

params_schema:
  {"params":[{"name":"client","desc":"cliente autenticado BQClient obtenido con bq_auth"},{"name":"dataset_id","desc":"ID del dataset que contiene la tabla"},{"name":"table_id","desc":"nombre (ID) de la tabla a actualizar"},{"name":"schema","desc":"schema completo nuevo (columnas existentes + nuevas al final). BigQuery SOLO permite agregar columnas, no eliminar ni renombrar. None = no modificar"},{"name":"description","desc":"nueva descripcion de la tabla. None = no modificar"},{"name":"labels","desc":"nuevas etiquetas clave-valor (reemplaza las existentes). None = no modificar"}],"output":"dict con la metadata actualizada de la tabla (misma estructura que bq_get_table)"}


id: all_of_py_core
name: all_of
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def all_of(xs: list, pred: callable) -> bool
description: Retorna True si todos los elementos de la lista cumplen el predicado.
tags: ["list","functional","predicate","all","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = all_of([2, 4, 6], lambda n: n % 2 == 0)
  # True
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Retorna True para lista vacia (vacuamente verdadero). Cortocircuita al primer False.
content_hash: c4adf0c65600e1da0e7c05c9e30f64c684ee40e5ba09ff04a2781eb8d510556e
code:
  """Core functional programming utilities — pure functions for list/collection operations."""

  import hashlib
  import re
  from functools import reduce as _reduce
  from typing import Any, Callable, Dict, List, Optional, Tuple


  def filter_list(xs: list, pred: Callable) -> list:
      """Filter list by predicate. Does not mutate the original."""
      return [x for x in xs if pred(x)]


  def map_list(xs: list, fn: Callable) -> list:
      """Map function over list. Does not mutate the original."""
      return [fn(x) for x in xs]


  def reduce_list(xs: list, initial: Any, fn: Callable) -> Any:
      """Reduce list with accumulator. fn(acc, x) -> acc."""
      return _reduce(fn, xs, initial)


  def flat_map(xs: list, fn: Callable) -> list:
      """Map function over list then flatten one level."""
      result = []
      for x in xs:
          result.extend(fn(x))
      return result


  def flatten(xss: list) -> list:
      """Flatten a list of lists one level."""
      result = []
      for xs in xss:
          result.extend(xs)
      return result


  def chunk(xs: list, size: int) -> list:
      """Split list into chunks of given size. Last chunk may be smaller."""
      if size <= 0:
          return []
      return [xs[i : i + size] for i in range(0, len(xs), size)]


  def take(xs: list, n: int) -> list:
      """Take first n elements from list."""
      return xs[:n]


  def drop(xs: list, n: int) -> list:
      """Drop first n elements from list."""
      return xs[n:]


  def unique(xs: list) -> list:
      """Remove duplicates preserving order. Uses identity for hashable elements."""
      seen = set()
      result = []
      for x in xs:
          if x not in seen:
              seen.add(x)
              result.append(x)
      return result


  def group_by(xs: list, key_fn: Callable) -> Dict:
      """Group elements by key function. Returns dict of key -> list."""
      groups: Dict = {}
      for x in xs:
          k = key_fn(x)
          if k not in groups:
              groups[k] = []
          groups[k].append(x)
      return groups


  def partition(xs: list, pred: Callable) -> Tuple[list, list]:
      """Split list into (matches, non_matches) based on predicate."""
      matches = []
      non_matches = []
      for x in xs:
          if pred(x):
              matches.append(x)
          else:
              non_matches.append(x)
      return (matches, non_matches)


  def find(xs: list, pred: Callable) -> Any:
      """Find first element matching predicate. Returns None if not found."""
      for x in xs:
          if pred(x):
              return x
      return None


  def find_index(xs: list, pred: Callable) -> int:
      """Find index of first element matching predicate. Returns -1 if not found."""
      for i, x in enumerate(xs):
          if pred(x):
              return i
      return -1


  def zip_with(xs: list, ys: list, fn: Callable) -> list:
      """Zip two lists with a combining function. Stops at shorter list."""
      return [fn(x, y) for x, y in zip(xs, ys)]


  def all_of(xs: list, pred: Callable) -> bool:
      """Return True if all elements match predicate."""
      return all(pred(x) for x in xs)


  def any_of(xs: list, pred: Callable) -> bool:
      """Return True if any element matches predicate."""
      return any(pred(x) for x in xs)


  def pipe(value: Any, *fns: Callable) -> Any:
      """Pipe a value through a sequence of functions left-to-right."""
      result = value
      for fn in fns:
          result = fn(result)
      return result


  def compose(*fns: Callable) -> Callable:
      """Compose functions right-to-left. compose(f, g)(x) == f(g(x))."""
      def composed(x: Any) -> Any:
          result = x
          for fn in reversed(fns):
              result = fn(result)
          return result
      return composed


  # ── Tree manipulation ────────────────────────────────────────────────────────


  def flatten_tree(structure: Any) -> List[Dict]:
      """Flatten a hierarchical tree (dict with 'nodes') to a list without children."""
      import copy
      if isinstance(structure, dict):
          node = copy.deepcopy(structure)
          node.pop('nodes', None)
          nodes = [node]
          for key in list(structure.keys()):
              if 'nodes' in key:
                  nodes.extend(flatten_tree(structure[key]))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(flatten_tree(item))
          return nodes
      return []


  def tree_to_flat_list(structure: Any) -> List[Dict]:
      """Convert hierarchical tree to flat list preserving DFS order (keeps internal nodes)."""
      if isinstance(structure, dict):
          nodes = [structure]
          if 'nodes' in structure:
              nodes.extend(tree_to_flat_list(structure['nodes']))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(tree_to_flat_list(item))
          return nodes
      return []


  def get_leaf_nodes(structure: Any) -> List[Dict]:
      """Extract only leaf nodes (no children) from a hierarchical tree."""
      import copy
      if isinstance(structure, dict):
          if not structure.get('nodes'):
              node = copy.deepcopy(structure)
              node.pop('nodes', None)
              return [node]
          leaf_nodes = []
          for key in list(structure.keys()):
              if 'nodes' in key:
                  leaf_nodes.extend(get_leaf_nodes(structure[key]))
          return leaf_nodes
      elif isinstance(structure, list):
          leaf_nodes = []
          for item in structure:
              leaf_nodes.extend(get_leaf_nodes(item))
          return leaf_nodes
      return []


  def write_node_ids(data: Any, node_id: int = 0) -> int:
      """Assign sequential zero-padded IDs (0001, 0002...) to all nodes in a tree. Returns next counter."""
      if isinstance(data, dict):
          data['node_id'] = str(node_id).zfill(4)
          node_id += 1
          for key in list(data.keys()):
              if 'nodes' in key:
                  node_id = write_node_ids(data[key], node_id)
      elif isinstance(data, list):
          for item in data:
              node_id = write_node_ids(item, node_id)
      return node_id


  def list_to_tree(data: List[Dict]) -> List[Dict]:
      """Convert flat list with structure codes ('1.2.3') to nested tree."""
      def get_parent_structure(structure):
          if not structure:
              return None
          parts = str(structure).split('.')
          return '.'.join(parts[:-1]) if len(parts) > 1 else None

      nodes = {}
      root_nodes = []

      for item in data:
          structure = item.get('structure')
          node = {
              'title': item.get('title'),
              'start_index': item.get('start_index'),
              'end_index': item.get('end_index'),
              'nodes': []
          }
          nodes[structure] = node
          parent_structure = get_parent_structure(structure)

          if parent_structure and parent_structure in nodes:
              nodes[parent_structure]['nodes'].append(node)
          else:
              root_nodes.append(node)

      def clean_node(node):
          if not node['nodes']:
              del node['nodes']
          else:
              for child in node['nodes']:
                  clean_node(child)
          return node

      return [clean_node(node) for node in root_nodes]


  def remove_tree_fields(data: Any, fields: List[str] = None) -> Any:
      """Recursively remove specified fields from a tree (dict/list)."""
      if fields is None:
          fields = ['text']
      if isinstance(data, dict):
          return {k: remove_tree_fields(v, fields) for k, v in data.items() if k not in fields}
      elif isinstance(data, list):
          return [remove_tree_fields(item, fields) for item in data]
      return data


  def format_tree_structure(structure: Any, order: List[str] = None) -> Any:
      """Reorder fields of each node in a tree according to specified key order."""
      if not order:
          return structure
      if isinstance(structure, dict):
          if 'nodes' in structure:
              structure['nodes'] = format_tree_structure(structure['nodes'], order)
          if not structure.get('nodes'):
              structure.pop('nodes', None)
          return {key: structure[key] for key in order if key in structure}
      elif isinstance(structure, list):
          return [format_tree_structure(item, order) for item in structure]
      return structure


  def create_node_mapping(tree: List[Dict]) -> Dict[str, Dict]:
      """Create flat dict mapping node_id to node for O(1) lookup."""
      mapping = {}
      def _traverse(nodes):
          for node in nodes:
              if node.get('node_id'):
                  mapping[node['node_id']] = node
              if node.get('nodes'):
                  _traverse(node['nodes'])
      _traverse(tree)
      return mapping


  # ── Text / JSON extraction ───────────────────────────────────────────────────


  def extract_json_from_llm(content: str) -> Dict:
      """Extract and parse JSON from LLM responses. Handles ```json blocks, trailing commas, None->null."""
      import json
      try:
          start_idx = content.find("```json")
          if start_idx != -1:
              start_idx += 7
              end_idx = content.rfind("```")
              json_content = content[start_idx:end_idx].strip()
          else:
              json_content = content.strip()

          json_content = json_content.replace('None', 'null')
          json_content = json_content.replace('\n', ' ').replace('\r', ' ')
          json_content = ' '.join(json_content.split())

          return json.loads(json_content)
      except (json.JSONDecodeError, Exception):
          try:
              json_content = json_content.replace(',]', ']').replace(',}', '}')
              return json.loads(json_content)
          except Exception:
              return {}


  def parse_page_range(pages: str) -> List[int]:
      """Parse page range string ('5-7', '3,8', '12') into sorted list of unique ints."""
      result = []
      for part in pages.split(','):
          part = part.strip()
          if '-' in part:
              start, end = int(part.split('-', 1)[0].strip()), int(part.split('-', 1)[1].strip())
              if start > end:
                  raise ValueError(f"Invalid range '{part}': start must be <= end")
              result.extend(range(start, end + 1))
          else:
              result.append(int(part))
      return sorted(set(result))


  # ── Markdown parsing ─────────────────────────────────────────────────────────


  def extract_markdown_headers(markdown_content: str) -> Tuple[List[Dict], List[str]]:
      """Extract all headers (h1-h6) from markdown with line numbers, skipping code blocks."""
      import re
      header_pattern = r'^(#{1,6})\s+(.+)$'
      code_block_pattern = r'^```'
      node_list = []
      lines = markdown_content.split('\n')
      in_code_block = False

      for line_num, line in enumerate(lines, 1):
          stripped_line = line.strip()
          if re.match(code_block_pattern, stripped_line):
              in_code_block = not in_code_block
              continue
          if not stripped_line:
              continue
          if not in_code_block:
              match = re.match(header_pattern, stripped_line)
              if match:
                  level = len(match.group(1))
                  title = match.group(2).strip()
                  node_list.append({'title': title, 'level': level, 'line_num': line_num})

      return node_list, lines


  def build_tree_from_headers(node_list: List[Dict]) -> List[Dict]:
      """Build nested tree from flat list of headers with levels (h1>h2>h3)."""
      if not node_list:
          return []

      stack = []
      root_nodes = []
      node_counter = 1

      for node in node_list:
          current_level = node['level']
          tree_node = {
              'title': node['title'],
              'node_id': str(node_counter).zfill(4),
              'line_num': node['line_num'],
              'nodes': []
          }
          node_counter += 1

          while stack and stack[-1][1] >= current_level:
              stack.pop()

          if not stack:
              root_nodes.append(tree_node)
          else:
              parent_node, _ = stack[-1]
              parent_node['nodes'].append(tree_node)

          stack.append((tree_node, current_level))

      def clean_empty_nodes(nodes):
          for n in nodes:
              if n['nodes']:
                  clean_empty_nodes(n['nodes'])
              else:
                  del n['nodes']
          return nodes

      return clean_empty_nodes(root_nodes)


  # ── Pagination / chunking ────────────────────────────────────────────────────


  def page_list_to_groups(page_contents: List[str], token_lengths: List[int],
                          max_tokens: int = 20000, overlap_pages: int = 1) -> List[str]:
      """Group pages into text chunks respecting token limit with configurable overlap."""
      import math
      num_tokens = sum(token_lengths)

      if num_tokens <= max_tokens:
          return ["".join(page_contents)]

      subsets = []
      current_subset = []
      current_token_count = 0

      expected_parts = math.ceil(num_tokens / max_tokens)
      avg_tokens = math.ceil(((num_tokens / expected_parts) + max_tokens) / 2)

      for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)):
          if current_token_count + page_tokens > avg_tokens:
              subsets.append(''.join(current_subset))
              overlap_start = max(i - overlap_pages, 0)
              current_subset = list(page_contents[overlap_start:i])
              current_token_count = sum(token_lengths[overlap_start:i])

          current_subset.append(page_content)
          current_token_count += page_tokens

      if current_subset:
          subsets.append(''.join(current_subset))

      return subsets


  def calculate_page_offset(pairs: List[Dict]) -> int:
      """Calculate offset between logical page numbers and physical indices using reference pairs."""
      differences = []
      for pair in pairs:
          try:
              difference = pair['physical_index'] - pair['page']
              differences.append(difference)
          except (KeyError, TypeError):
              continue

      if not differences:
          return 0

      counts: Dict[int, int] = {}
      for diff in differences:
          counts[diff] = counts.get(diff, 0) + 1

      return max(counts.items(), key=lambda x: x[1])[0]


  # ── Text preprocessing ───────────────────────────────────────────────────────


  def preprocess_text(text: str) -> str:
      """Normalize whitespace and newlines in raw text.

      Args:
          text: Raw text to normalize.

      Returns:
          Normalized text with consistent newlines, stripped lines, and no
          excessive blank lines.
      """
      # Normalize line endings: \r\n and \r -> \n
      text = text.replace('\r\n', '\n').replace('\r', '\n')
      # Reduce 3+ consecutive newlines to at most 2
      text = re.sub(r'\n{3,}', '\n\n', text)
      # Strip whitespace from each line
      text = '\n'.join(line.strip() for line in text.split('\n'))
      # Strip globally
      return text.strip()


  def get_text_stats(text: str) -> dict:
      """Compute basic statistics of a text: characters, lines, words.

      Args:
          text: Input text to analyze.

      Returns:
          Dict with keys total_chars (int), total_lines (int), total_words (int).
      """
      return {
          'total_chars': len(text),
          'total_lines': text.count('\n') + 1,
          'total_words': len(text.split()),
      }


  # ── Git URL parsing ──────────────────────────────────────────────────────────

  _DEFAULT_GIT_HOSTS = ["github.com", "gitlab.com"]


  def _sanitize_git_segment(segment: str) -> str:
      """Strip .git suffix then keep only [a-zA-Z0-9_-] chars."""
      if segment.endswith(".git"):
          segment = segment[:-4]
      return re.sub(r"[^a-zA-Z0-9_\-]", "", segment)


  def parse_git_url(url: str, known_hosts: Optional[List[str]] = None) -> Optional[str]:
      """Parse a code-hosting URL and return the 'org/repo' path component.

      Supports HTTPS, HTTP, git://, ssh:// and SSH shorthand (git@host:path).
      Returns None if the URL does not match any known host or is malformed.

      Args:
          url: Repository URL in any supported format.
          known_hosts: List of accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          'org/repo' string or None.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      if url.startswith("git@"):
          # git@github.com:org/repo.git
          rest = url[len("git@"):]
          if ":" not in rest:
              return None
          host, path = rest.split(":", 1)
          if host not in hosts:
              return None
          segments = [s for s in path.split("/") if s]
          if len(segments) < 2:
              return None
          org = _sanitize_git_segment(segments[0])
          repo = _sanitize_git_segment(segments[1])
          if not org or not repo:
              return None
          return f"{org}/{repo}"

      for prefix in ("http://", "https://", "git://", "ssh://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              netloc = parsed.hostname or ""
              if netloc not in hosts:
                  return None
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) < 2:
                  return None
              org = _sanitize_git_segment(segments[0])
              repo = _sanitize_git_segment(segments[1])
              if not org or not repo:
                  return None
              return f"{org}/{repo}"

      return None


  def is_git_repo_url(url: str, known_hosts: Optional[List[str]] = None) -> bool:
      """Return True only if url points to a clonable git repository.

      Accepts org/repo and org/repo/tree/<ref> paths.
      Rejects paths that navigate to sub-resources (issues, blobs, PRs, etc.).

      Args:
          url: URL to verify.
          known_hosts: Accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          True if url is a clonable repository URL.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      # SSH shorthand — always repo-level if host matches
      if url.startswith("git@"):
          rest = url[len("git@"):]
          if ":" not in rest:
              return False
          host, _ = rest.split(":", 1)
          return host in hosts

      # git:// and ssh:// — always repo-level if host matches
      for prefix in ("ssh://", "git://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              return (parsed.hostname or "") in hosts

      # http:// and https:// — must have exactly org/repo or org/repo/tree/<ref>
      for prefix in ("http://", "https://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              if (parsed.hostname or "") not in hosts:
                  return False
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) == 2:
                  return True
              if len(segments) == 4 and segments[2] == "tree":
                  return True
              return False

      return False


  def validate_git_ssh_uri(url: str) -> None:
      """Validate a git SSH URI of the form git@host:path.

      Raises ValueError with a descriptive message if the URI is malformed.

      Args:
          url: URI string to validate.

      Raises:
          ValueError: If the URI does not conform to git SSH format.
      """
      if not url.startswith("git@"):
          raise ValueError(f"git SSH URI must start with 'git@', got: {url!r}")
      rest = url[len("git@"):]
      if ":" not in rest:
          raise ValueError(f"git SSH URI must contain ':', got: {url!r}")
      _, path = rest.split(":", 1)
      if not path:
          raise ValueError(f"git SSH URI must have a non-empty path after ':', got: {url!r}")


  # ---------------------------------------------------------------------------
  # Markdown parsing utilities
  # ---------------------------------------------------------------------------


  def extract_frontmatter(content: str) -> Tuple[str, Optional[Dict]]:
      """Extract YAML frontmatter delimited by '---' from the start of a markdown string.

      Args:
          content: Raw markdown string, optionally starting with YAML frontmatter.

      Returns:
          Tuple of (content_without_frontmatter, frontmatter_dict).
          frontmatter_dict is None when no frontmatter is found.
      """
      pattern = re.compile(r'^---\n(.*?)\n---\n', re.DOTALL)
      match = pattern.match(content)
      if not match:
          return content, None

      raw = match.group(1)
      remaining = content[match.end():]

      try:
          import yaml  # type: ignore
          data = yaml.safe_load(raw)
          if not isinstance(data, dict):
              data = None
      except Exception:
          # Fallback: simple key: value parser (no yaml dependency)
          data = {}
          for line in raw.splitlines():
              if ':' in line:
                  key, _, value = line.partition(':')
                  data[key.strip()] = value.strip()

      return remaining, data


  def find_headings(content: str) -> List[Tuple[int, int, str, int]]:
      """Find all markdown headings (# to ######), excluding those inside code blocks,
      HTML comments, and indented blocks.

      Args:
          content: Markdown text to search.

      Returns:
          List of (start_pos, end_pos, title, level) for each heading found.
      """
      excluded: List[Tuple[int, int]] = []

      # Code blocks (triple backtick)
      for m in re.finditer(r'```.*?```', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # HTML comments
      for m in re.finditer(r'<!--.*?-->', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # Indented blocks (lines starting with 4 spaces or a tab)
      for m in re.finditer(r'^(    |\t).+$', content, re.MULTILINE):
          excluded.append((m.start(), m.end()))

      def is_excluded(pos: int) -> bool:
          return any(start <= pos < end for start, end in excluded)

      results: List[Tuple[int, int, str, int]] = []
      for m in re.finditer(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE):
          # Skip escaped headings (\#)
          before = content[m.start() - 1] if m.start() > 0 else ''
          if before == '\\':
              continue
          if is_excluded(m.start()):
              continue
          level = len(m.group(1))
          title = m.group(2).strip()
          results.append((m.start(), m.end(), title, level))

      return results


  def estimate_token_count(content: str) -> int:
      """Estimate token count without a tokenizer.

      CJK characters count as ~0.7 tokens each; other non-whitespace characters
      count as ~0.3 tokens each.

      Args:
          content: Text to estimate.

      Returns:
          Estimated integer token count.
      """
      cjk = re.findall(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', content)
      without_cjk = re.sub(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', '', content)
      others = re.findall(r'\S', without_cjk)
      return int(len(cjk) * 0.7 + len(others) * 0.3)


  def smart_split_content(
      content: str,
      max_tokens: int = 1024,
      max_chars: int = 8000,
  ) -> List[str]:
      """Split large content into parts respecting token and character limits.

      Splits by paragraphs (double newline). If a single paragraph exceeds the
      limit it is force-cut into chunks of max_chars.

      Args:
          content: Text to split.
          max_tokens: Maximum estimated tokens per part.
          max_chars: Maximum characters per part.

      Returns:
          List of string parts.
      """
      paragraphs = content.split('\n\n')
      parts: List[str] = []
      current_parts: List[str] = []
      current_tokens = 0
      current_chars = 0

      def flush() -> None:
          if current_parts:
              parts.append('\n\n'.join(current_parts))
              current_parts.clear()

      for para in paragraphs:
          para_tokens = estimate_token_count(para)
          para_chars = len(para)

          # Single paragraph exceeds limits — force-cut it
          if para_tokens > max_tokens or para_chars > max_chars:
              flush()
              current_tokens = 0
              current_chars = 0
              for i in range(0, len(para), max_chars):
                  parts.append(para[i:i + max_chars])
              continue

          # Would exceed limits if added — flush first
          if (current_tokens + para_tokens > max_tokens or
                  current_chars + para_chars > max_chars):
              flush()
              current_tokens = 0
              current_chars = 0

          current_parts.append(para)
          current_tokens += para_tokens
          current_chars += para_chars

      flush()
      return parts if parts else [content]


  def sanitize_for_path(text: str, max_length: int = 50) -> str:
      """Convert text to a safe string for use in file paths.

      Keeps word characters, CJK characters, spaces and hyphens. Replaces spaces
      with underscores. Truncates with a sha256 suffix if the result exceeds
      max_length.

      Args:
          text: Input text to sanitize.
          max_length: Maximum length of the returned string.

      Returns:
          Safe path-friendly string.
      """
      cleaned = re.sub(
          r'[^\w\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af \-]',
          '',
          text,
      )
      cleaned = cleaned.replace(' ', '_').strip('_')

      if not cleaned:
          return 'section'

      if len(cleaned) <= max_length:
          return cleaned

      suffix = '_' + hashlib.sha256(text.encode()).hexdigest()[:8]
      return cleaned[:max_length - len(suffix)] + suffix

params_schema:
  {"params":[{"name":"xs","desc":"lista de elementos a evaluar"},{"name":"pred","desc":"predicado (funcion que retorna bool) para evaluar cada elemento"}],"output":"booleano: True si todos los elementos cumplen el predicado, False en caso contrario"}


id: any_of_py_core
name: any_of
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def any_of(xs: list, pred: callable) -> bool
description: Retorna True si al menos un elemento de la lista cumple el predicado.
tags: ["list","functional","predicate","any","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = any_of([1, 3, 5, 4], lambda n: n % 2 == 0)
  # True
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Retorna False para lista vacia. Cortocircuita al primer True.
content_hash: 81eaac976d537b622956a9b2290b9f24101a3a19d0285fa415a2cad2637927e0
code:
  """Core functional programming utilities — pure functions for list/collection operations."""

  import hashlib
  import re
  from functools import reduce as _reduce
  from typing import Any, Callable, Dict, List, Optional, Tuple


  def filter_list(xs: list, pred: Callable) -> list:
      """Filter list by predicate. Does not mutate the original."""
      return [x for x in xs if pred(x)]


  def map_list(xs: list, fn: Callable) -> list:
      """Map function over list. Does not mutate the original."""
      return [fn(x) for x in xs]


  def reduce_list(xs: list, initial: Any, fn: Callable) -> Any:
      """Reduce list with accumulator. fn(acc, x) -> acc."""
      return _reduce(fn, xs, initial)


  def flat_map(xs: list, fn: Callable) -> list:
      """Map function over list then flatten one level."""
      result = []
      for x in xs:
          result.extend(fn(x))
      return result


  def flatten(xss: list) -> list:
      """Flatten a list of lists one level."""
      result = []
      for xs in xss:
          result.extend(xs)
      return result


  def chunk(xs: list, size: int) -> list:
      """Split list into chunks of given size. Last chunk may be smaller."""
      if size <= 0:
          return []
      return [xs[i : i + size] for i in range(0, len(xs), size)]


  def take(xs: list, n: int) -> list:
      """Take first n elements from list."""
      return xs[:n]


  def drop(xs: list, n: int) -> list:
      """Drop first n elements from list."""
      return xs[n:]


  def unique(xs: list) -> list:
      """Remove duplicates preserving order. Uses identity for hashable elements."""
      seen = set()
      result = []
      for x in xs:
          if x not in seen:
              seen.add(x)
              result.append(x)
      return result


  def group_by(xs: list, key_fn: Callable) -> Dict:
      """Group elements by key function. Returns dict of key -> list."""
      groups: Dict = {}
      for x in xs:
          k = key_fn(x)
          if k not in groups:
              groups[k] = []
          groups[k].append(x)
      return groups


  def partition(xs: list, pred: Callable) -> Tuple[list, list]:
      """Split list into (matches, non_matches) based on predicate."""
      matches = []
      non_matches = []
      for x in xs:
          if pred(x):
              matches.append(x)
          else:
              non_matches.append(x)
      return (matches, non_matches)


  def find(xs: list, pred: Callable) -> Any:
      """Find first element matching predicate. Returns None if not found."""
      for x in xs:
          if pred(x):
              return x
      return None


  def find_index(xs: list, pred: Callable) -> int:
      """Find index of first element matching predicate. Returns -1 if not found."""
      for i, x in enumerate(xs):
          if pred(x):
              return i
      return -1


  def zip_with(xs: list, ys: list, fn: Callable) -> list:
      """Zip two lists with a combining function. Stops at shorter list."""
      return [fn(x, y) for x, y in zip(xs, ys)]


  def all_of(xs: list, pred: Callable) -> bool:
      """Return True if all elements match predicate."""
      return all(pred(x) for x in xs)


  def any_of(xs: list, pred: Callable) -> bool:
      """Return True if any element matches predicate."""
      return any(pred(x) for x in xs)


  def pipe(value: Any, *fns: Callable) -> Any:
      """Pipe a value through a sequence of functions left-to-right."""
      result = value
      for fn in fns:
          result = fn(result)
      return result


  def compose(*fns: Callable) -> Callable:
      """Compose functions right-to-left. compose(f, g)(x) == f(g(x))."""
      def composed(x: Any) -> Any:
          result = x
          for fn in reversed(fns):
              result = fn(result)
          return result
      return composed


  # ── Tree manipulation ────────────────────────────────────────────────────────


  def flatten_tree(structure: Any) -> List[Dict]:
      """Flatten a hierarchical tree (dict with 'nodes') to a list without children."""
      import copy
      if isinstance(structure, dict):
          node = copy.deepcopy(structure)
          node.pop('nodes', None)
          nodes = [node]
          for key in list(structure.keys()):
              if 'nodes' in key:
                  nodes.extend(flatten_tree(structure[key]))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(flatten_tree(item))
          return nodes
      return []


  def tree_to_flat_list(structure: Any) -> List[Dict]:
      """Convert hierarchical tree to flat list preserving DFS order (keeps internal nodes)."""
      if isinstance(structure, dict):
          nodes = [structure]
          if 'nodes' in structure:
              nodes.extend(tree_to_flat_list(structure['nodes']))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(tree_to_flat_list(item))
          return nodes
      return []


  def get_leaf_nodes(structure: Any) -> List[Dict]:
      """Extract only leaf nodes (no children) from a hierarchical tree."""
      import copy
      if isinstance(structure, dict):
          if not structure.get('nodes'):
              node = copy.deepcopy(structure)
              node.pop('nodes', None)
              return [node]
          leaf_nodes = []
          for key in list(structure.keys()):
              if 'nodes' in key:
                  leaf_nodes.extend(get_leaf_nodes(structure[key]))
          return leaf_nodes
      elif isinstance(structure, list):
          leaf_nodes = []
          for item in structure:
              leaf_nodes.extend(get_leaf_nodes(item))
          return leaf_nodes
      return []


  def write_node_ids(data: Any, node_id: int = 0) -> int:
      """Assign sequential zero-padded IDs (0001, 0002...) to all nodes in a tree. Returns next counter."""
      if isinstance(data, dict):
          data['node_id'] = str(node_id).zfill(4)
          node_id += 1
          for key in list(data.keys()):
              if 'nodes' in key:
                  node_id = write_node_ids(data[key], node_id)
      elif isinstance(data, list):
          for item in data:
              node_id = write_node_ids(item, node_id)
      return node_id


  def list_to_tree(data: List[Dict]) -> List[Dict]:
      """Convert flat list with structure codes ('1.2.3') to nested tree."""
      def get_parent_structure(structure):
          if not structure:
              return None
          parts = str(structure).split('.')
          return '.'.join(parts[:-1]) if len(parts) > 1 else None

      nodes = {}
      root_nodes = []

      for item in data:
          structure = item.get('structure')
          node = {
              'title': item.get('title'),
              'start_index': item.get('start_index'),
              'end_index': item.get('end_index'),
              'nodes': []
          }
          nodes[structure] = node
          parent_structure = get_parent_structure(structure)

          if parent_structure and parent_structure in nodes:
              nodes[parent_structure]['nodes'].append(node)
          else:
              root_nodes.append(node)

      def clean_node(node):
          if not node['nodes']:
              del node['nodes']
          else:
              for child in node['nodes']:
                  clean_node(child)
          return node

      return [clean_node(node) for node in root_nodes]


  def remove_tree_fields(data: Any, fields: List[str] = None) -> Any:
      """Recursively remove specified fields from a tree (dict/list)."""
      if fields is None:
          fields = ['text']
      if isinstance(data, dict):
          return {k: remove_tree_fields(v, fields) for k, v in data.items() if k not in fields}
      elif isinstance(data, list):
          return [remove_tree_fields(item, fields) for item in data]
      return data


  def format_tree_structure(structure: Any, order: List[str] = None) -> Any:
      """Reorder fields of each node in a tree according to specified key order."""
      if not order:
          return structure
      if isinstance(structure, dict):
          if 'nodes' in structure:
              structure['nodes'] = format_tree_structure(structure['nodes'], order)
          if not structure.get('nodes'):
              structure.pop('nodes', None)
          return {key: structure[key] for key in order if key in structure}
      elif isinstance(structure, list):
          return [format_tree_structure(item, order) for item in structure]
      return structure


  def create_node_mapping(tree: List[Dict]) -> Dict[str, Dict]:
      """Create flat dict mapping node_id to node for O(1) lookup."""
      mapping = {}
      def _traverse(nodes):
          for node in nodes:
              if node.get('node_id'):
                  mapping[node['node_id']] = node
              if node.get('nodes'):
                  _traverse(node['nodes'])
      _traverse(tree)
      return mapping


  # ── Text / JSON extraction ───────────────────────────────────────────────────


  def extract_json_from_llm(content: str) -> Dict:
      """Extract and parse JSON from LLM responses. Handles ```json blocks, trailing commas, None->null."""
      import json
      try:
          start_idx = content.find("```json")
          if start_idx != -1:
              start_idx += 7
              end_idx = content.rfind("```")
              json_content = content[start_idx:end_idx].strip()
          else:
              json_content = content.strip()

          json_content = json_content.replace('None', 'null')
          json_content = json_content.replace('\n', ' ').replace('\r', ' ')
          json_content = ' '.join(json_content.split())

          return json.loads(json_content)
      except (json.JSONDecodeError, Exception):
          try:
              json_content = json_content.replace(',]', ']').replace(',}', '}')
              return json.loads(json_content)
          except Exception:
              return {}


  def parse_page_range(pages: str) -> List[int]:
      """Parse page range string ('5-7', '3,8', '12') into sorted list of unique ints."""
      result = []
      for part in pages.split(','):
          part = part.strip()
          if '-' in part:
              start, end = int(part.split('-', 1)[0].strip()), int(part.split('-', 1)[1].strip())
              if start > end:
                  raise ValueError(f"Invalid range '{part}': start must be <= end")
              result.extend(range(start, end + 1))
          else:
              result.append(int(part))
      return sorted(set(result))


  # ── Markdown parsing ─────────────────────────────────────────────────────────


  def extract_markdown_headers(markdown_content: str) -> Tuple[List[Dict], List[str]]:
      """Extract all headers (h1-h6) from markdown with line numbers, skipping code blocks."""
      import re
      header_pattern = r'^(#{1,6})\s+(.+)$'
      code_block_pattern = r'^```'
      node_list = []
      lines = markdown_content.split('\n')
      in_code_block = False

      for line_num, line in enumerate(lines, 1):
          stripped_line = line.strip()
          if re.match(code_block_pattern, stripped_line):
              in_code_block = not in_code_block
              continue
          if not stripped_line:
              continue
          if not in_code_block:
              match = re.match(header_pattern, stripped_line)
              if match:
                  level = len(match.group(1))
                  title = match.group(2).strip()
                  node_list.append({'title': title, 'level': level, 'line_num': line_num})

      return node_list, lines


  def build_tree_from_headers(node_list: List[Dict]) -> List[Dict]:
      """Build nested tree from flat list of headers with levels (h1>h2>h3)."""
      if not node_list:
          return []

      stack = []
      root_nodes = []
      node_counter = 1

      for node in node_list:
          current_level = node['level']
          tree_node = {
              'title': node['title'],
              'node_id': str(node_counter).zfill(4),
              'line_num': node['line_num'],
              'nodes': []
          }
          node_counter += 1

          while stack and stack[-1][1] >= current_level:
              stack.pop()

          if not stack:
              root_nodes.append(tree_node)
          else:
              parent_node, _ = stack[-1]
              parent_node['nodes'].append(tree_node)

          stack.append((tree_node, current_level))

      def clean_empty_nodes(nodes):
          for n in nodes:
              if n['nodes']:
                  clean_empty_nodes(n['nodes'])
              else:
                  del n['nodes']
          return nodes

      return clean_empty_nodes(root_nodes)


  # ── Pagination / chunking ────────────────────────────────────────────────────


  def page_list_to_groups(page_contents: List[str], token_lengths: List[int],
                          max_tokens: int = 20000, overlap_pages: int = 1) -> List[str]:
      """Group pages into text chunks respecting token limit with configurable overlap."""
      import math
      num_tokens = sum(token_lengths)

      if num_tokens <= max_tokens:
          return ["".join(page_contents)]

      subsets = []
      current_subset = []
      current_token_count = 0

      expected_parts = math.ceil(num_tokens / max_tokens)
      avg_tokens = math.ceil(((num_tokens / expected_parts) + max_tokens) / 2)

      for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)):
          if current_token_count + page_tokens > avg_tokens:
              subsets.append(''.join(current_subset))
              overlap_start = max(i - overlap_pages, 0)
              current_subset = list(page_contents[overlap_start:i])
              current_token_count = sum(token_lengths[overlap_start:i])

          current_subset.append(page_content)
          current_token_count += page_tokens

      if current_subset:
          subsets.append(''.join(current_subset))

      return subsets


  def calculate_page_offset(pairs: List[Dict]) -> int:
      """Calculate offset between logical page numbers and physical indices using reference pairs."""
      differences = []
      for pair in pairs:
          try:
              difference = pair['physical_index'] - pair['page']
              differences.append(difference)
          except (KeyError, TypeError):
              continue

      if not differences:
          return 0

      counts: Dict[int, int] = {}
      for diff in differences:
          counts[diff] = counts.get(diff, 0) + 1

      return max(counts.items(), key=lambda x: x[1])[0]


  # ── Text preprocessing ───────────────────────────────────────────────────────


  def preprocess_text(text: str) -> str:
      """Normalize whitespace and newlines in raw text.

      Args:
          text: Raw text to normalize.

      Returns:
          Normalized text with consistent newlines, stripped lines, and no
          excessive blank lines.
      """
      # Normalize line endings: \r\n and \r -> \n
      text = text.replace('\r\n', '\n').replace('\r', '\n')
      # Reduce 3+ consecutive newlines to at most 2
      text = re.sub(r'\n{3,}', '\n\n', text)
      # Strip whitespace from each line
      text = '\n'.join(line.strip() for line in text.split('\n'))
      # Strip globally
      return text.strip()


  def get_text_stats(text: str) -> dict:
      """Compute basic statistics of a text: characters, lines, words.

      Args:
          text: Input text to analyze.

      Returns:
          Dict with keys total_chars (int), total_lines (int), total_words (int).
      """
      return {
          'total_chars': len(text),
          'total_lines': text.count('\n') + 1,
          'total_words': len(text.split()),
      }


  # ── Git URL parsing ──────────────────────────────────────────────────────────

  _DEFAULT_GIT_HOSTS = ["github.com", "gitlab.com"]


  def _sanitize_git_segment(segment: str) -> str:
      """Strip .git suffix then keep only [a-zA-Z0-9_-] chars."""
      if segment.endswith(".git"):
          segment = segment[:-4]
      return re.sub(r"[^a-zA-Z0-9_\-]", "", segment)


  def parse_git_url(url: str, known_hosts: Optional[List[str]] = None) -> Optional[str]:
      """Parse a code-hosting URL and return the 'org/repo' path component.

      Supports HTTPS, HTTP, git://, ssh:// and SSH shorthand (git@host:path).
      Returns None if the URL does not match any known host or is malformed.

      Args:
          url: Repository URL in any supported format.
          known_hosts: List of accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          'org/repo' string or None.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      if url.startswith("git@"):
          # git@github.com:org/repo.git
          rest = url[len("git@"):]
          if ":" not in rest:
              return None
          host, path = rest.split(":", 1)
          if host not in hosts:
              return None
          segments = [s for s in path.split("/") if s]
          if len(segments) < 2:
              return None
          org = _sanitize_git_segment(segments[0])
          repo = _sanitize_git_segment(segments[1])
          if not org or not repo:
              return None
          return f"{org}/{repo}"

      for prefix in ("http://", "https://", "git://", "ssh://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              netloc = parsed.hostname or ""
              if netloc not in hosts:
                  return None
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) < 2:
                  return None
              org = _sanitize_git_segment(segments[0])
              repo = _sanitize_git_segment(segments[1])
              if not org or not repo:
                  return None
              return f"{org}/{repo}"

      return None


  def is_git_repo_url(url: str, known_hosts: Optional[List[str]] = None) -> bool:
      """Return True only if url points to a clonable git repository.

      Accepts org/repo and org/repo/tree/<ref> paths.
      Rejects paths that navigate to sub-resources (issues, blobs, PRs, etc.).

      Args:
          url: URL to verify.
          known_hosts: Accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          True if url is a clonable repository URL.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      # SSH shorthand — always repo-level if host matches
      if url.startswith("git@"):
          rest = url[len("git@"):]
          if ":" not in rest:
              return False
          host, _ = rest.split(":", 1)
          return host in hosts

      # git:// and ssh:// — always repo-level if host matches
      for prefix in ("ssh://", "git://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              return (parsed.hostname or "") in hosts

      # http:// and https:// — must have exactly org/repo or org/repo/tree/<ref>
      for prefix in ("http://", "https://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              if (parsed.hostname or "") not in hosts:
                  return False
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) == 2:
                  return True
              if len(segments) == 4 and segments[2] == "tree":
                  return True
              return False

      return False


  def validate_git_ssh_uri(url: str) -> None:
      """Validate a git SSH URI of the form git@host:path.

      Raises ValueError with a descriptive message if the URI is malformed.

      Args:
          url: URI string to validate.

      Raises:
          ValueError: If the URI does not conform to git SSH format.
      """
      if not url.startswith("git@"):
          raise ValueError(f"git SSH URI must start with 'git@', got: {url!r}")
      rest = url[len("git@"):]
      if ":" not in rest:
          raise ValueError(f"git SSH URI must contain ':', got: {url!r}")
      _, path = rest.split(":", 1)
      if not path:
          raise ValueError(f"git SSH URI must have a non-empty path after ':', got: {url!r}")


  # ---------------------------------------------------------------------------
  # Markdown parsing utilities
  # ---------------------------------------------------------------------------


  def extract_frontmatter(content: str) -> Tuple[str, Optional[Dict]]:
      """Extract YAML frontmatter delimited by '---' from the start of a markdown string.

      Args:
          content: Raw markdown string, optionally starting with YAML frontmatter.

      Returns:
          Tuple of (content_without_frontmatter, frontmatter_dict).
          frontmatter_dict is None when no frontmatter is found.
      """
      pattern = re.compile(r'^---\n(.*?)\n---\n', re.DOTALL)
      match = pattern.match(content)
      if not match:
          return content, None

      raw = match.group(1)
      remaining = content[match.end():]

      try:
          import yaml  # type: ignore
          data = yaml.safe_load(raw)
          if not isinstance(data, dict):
              data = None
      except Exception:
          # Fallback: simple key: value parser (no yaml dependency)
          data = {}
          for line in raw.splitlines():
              if ':' in line:
                  key, _, value = line.partition(':')
                  data[key.strip()] = value.strip()

      return remaining, data


  def find_headings(content: str) -> List[Tuple[int, int, str, int]]:
      """Find all markdown headings (# to ######), excluding those inside code blocks,
      HTML comments, and indented blocks.

      Args:
          content: Markdown text to search.

      Returns:
          List of (start_pos, end_pos, title, level) for each heading found.
      """
      excluded: List[Tuple[int, int]] = []

      # Code blocks (triple backtick)
      for m in re.finditer(r'```.*?```', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # HTML comments
      for m in re.finditer(r'<!--.*?-->', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # Indented blocks (lines starting with 4 spaces or a tab)
      for m in re.finditer(r'^(    |\t).+$', content, re.MULTILINE):
          excluded.append((m.start(), m.end()))

      def is_excluded(pos: int) -> bool:
          return any(start <= pos < end for start, end in excluded)

      results: List[Tuple[int, int, str, int]] = []
      for m in re.finditer(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE):
          # Skip escaped headings (\#)
          before = content[m.start() - 1] if m.start() > 0 else ''
          if before == '\\':
              continue
          if is_excluded(m.start()):
              continue
          level = len(m.group(1))
          title = m.group(2).strip()
          results.append((m.start(), m.end(), title, level))

      return results


  def estimate_token_count(content: str) -> int:
      """Estimate token count without a tokenizer.

      CJK characters count as ~0.7 tokens each; other non-whitespace characters
      count as ~0.3 tokens each.

      Args:
          content: Text to estimate.

      Returns:
          Estimated integer token count.
      """
      cjk = re.findall(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', content)
      without_cjk = re.sub(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', '', content)
      others = re.findall(r'\S', without_cjk)
      return int(len(cjk) * 0.7 + len(others) * 0.3)


  def smart_split_content(
      content: str,
      max_tokens: int = 1024,
      max_chars: int = 8000,
  ) -> List[str]:
      """Split large content into parts respecting token and character limits.

      Splits by paragraphs (double newline). If a single paragraph exceeds the
      limit it is force-cut into chunks of max_chars.

      Args:
          content: Text to split.
          max_tokens: Maximum estimated tokens per part.
          max_chars: Maximum characters per part.

      Returns:
          List of string parts.
      """
      paragraphs = content.split('\n\n')
      parts: List[str] = []
      current_parts: List[str] = []
      current_tokens = 0
      current_chars = 0

      def flush() -> None:
          if current_parts:
              parts.append('\n\n'.join(current_parts))
              current_parts.clear()

      for para in paragraphs:
          para_tokens = estimate_token_count(para)
          para_chars = len(para)

          # Single paragraph exceeds limits — force-cut it
          if para_tokens > max_tokens or para_chars > max_chars:
              flush()
              current_tokens = 0
              current_chars = 0
              for i in range(0, len(para), max_chars):
                  parts.append(para[i:i + max_chars])
              continue

          # Would exceed limits if added — flush first
          if (current_tokens + para_tokens > max_tokens or
                  current_chars + para_chars > max_chars):
              flush()
              current_tokens = 0
              current_chars = 0

          current_parts.append(para)
          current_tokens += para_tokens
          current_chars += para_chars

      flush()
      return parts if parts else [content]


  def sanitize_for_path(text: str, max_length: int = 50) -> str:
      """Convert text to a safe string for use in file paths.

      Keeps word characters, CJK characters, spaces and hyphens. Replaces spaces
      with underscores. Truncates with a sha256 suffix if the result exceeds
      max_length.

      Args:
          text: Input text to sanitize.
          max_length: Maximum length of the returned string.

      Returns:
          Safe path-friendly string.
      """
      cleaned = re.sub(
          r'[^\w\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af \-]',
          '',
          text,
      )
      cleaned = cleaned.replace(' ', '_').strip('_')

      if not cleaned:
          return 'section'

      if len(cleaned) <= max_length:
          return cleaned

      suffix = '_' + hashlib.sha256(text.encode()).hexdigest()[:8]
      return cleaned[:max_length - len(suffix)] + suffix

params_schema:
  {"params":[{"name":"xs","desc":"lista de elementos a evaluar"},{"name":"pred","desc":"predicado (funcion que retorna bool) para evaluar cada elemento"}],"output":"booleano: True si al menos un elemento cumple el predicado, False en caso contrario"}


id: build_guide_prompt_py_core
name: build_guide_prompt
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["prompt","llm","guide","location","pois","interests","openai","ollama","context"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/build_guide_prompt_test.py
file_path: python/functions/core/build_guide_prompt.py
created_at: 2026-04-06T16:24:08Z
updated_at: 2026-04-06T16:24:08Z
props: []
emits: []
variant: []
content_hash: 4fc88d4a5e779f82b617f911546e7adae6899c918b7ffd0ece7485921aac73f9
signature:
  def build_guide_prompt(location: dict, pois: list[dict], interests: list[dict], user_query: str = "", lang: str = "es") -> list[dict]

description:
  Construye un prompt contextual para el LLM combinando ubicación actual, POIs cercanos rankeados e intereses del usuario. Retorna mensajes en formato OpenAI/Ollama.

example:
  ```python
  messages = build_guide_prompt(
      location={
          "display_name": "Puerta del Sol, Madrid",
          "city": "Madrid",
          "neighbourhood": "Sol",
          "street": "Puerta del Sol",
          "country": "España",
      },
      pois=[
          {
              "name": "Museo del Prado",
              "category": "museum",
              "score": 1.0,
              "matched_interests": ["arte"],
              "lat": 40.41,
              "lon": -3.70,
              "tags": {},
          }
      ],
      interests=[{"name": "arte", "keywords": ["museo"], "weight": 1.0}],
      user_query="¿Qué hay interesante por aquí?",
  )
  # [
  #   {"role": "system", "content": "Eres un guía local experto y amigable. El usuario está caminando por Madrid y le interesan: arte..."},
  #   {"role": "user", "content": "📍 Estoy en: Puerta del Sol, Madrid\nBarrio: Sol, Madrid\n\nLugares cercanos que me pueden interesar:\n- Museo del Prado (museum) — relevante para: arte\n\n¿Qué hay interesante por aquí?"}
  # ]
  ```

tests:
  ["con ubicacion completa pois y query","sin pois solo ubicacion","sin query usa default","max 5 pois","lang aparece en system","location sin neighbourhood"]

notes:
  Función pura — no hace I/O, no muta los inputs, determinista.

  El system message establece el rol del LLM como guía local con conocimiento de los intereses del usuario y el idioma de respuesta. Incluye instrucción de ser conciso (3-4 frases) y conversacional.

  El user message combina:
  - Ubicación completa del usuario (solo campos no vacíos)
  - Lista de hasta 5 POIs con nombre, categoría y los intereses que satisfacen
  - La query del usuario o un fallback genérico

  Los campos de `location` vacíos o ausentes se omiten automáticamente del prompt para no generar ruido. Compatible con el formato de salida de `nominatim_reverse_geocode_py_infra` y `match_pois_to_interests`.

code:
  """Construye un prompt contextual para el LLM combinando ubicación, POIs e intereses del usuario."""


  def build_guide_prompt(
      location: dict,
      pois: list[dict],
      interests: list[dict],
      user_query: str = "",
      lang: str = "es",
  ) -> list[dict]:
      """Construye un prompt contextual para el LLM como guía local.

      Combina la ubicación actual del usuario, POIs cercanos rankeados por relevancia
      e intereses del usuario para generar mensajes en formato OpenAI/Ollama.

      Args:
          location: Dict con info de ubicación de nominatim_reverse_geocode.
                    Campos esperados: display_name, street, neighbourhood, city, country, etc.
          pois: Lista de POIs rankeados con campos: name, category, score,
                matched_interests, lat, lon, tags. Ya filtrados y ordenados por score.
          interests: Lista de intereses del usuario. Cada uno con: name, keywords, weight.
          user_query: Pregunta opcional del usuario. Si está vacía, se usa un default.
          lang: Idioma para la respuesta del LLM (ej: "es", "en", "fr").

      Returns:
          Lista de mensajes en formato OpenAI/Ollama:
          [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}]
      """
      # Extraer nombre de intereses para el system prompt
      interest_names = [i.get("name", "") for i in interests if i.get("name")]
      interests_str = ", ".join(interest_names) if interest_names else "lugares de interés general"

      # Extraer ciudad para el system prompt
      city = location.get("city", "") or location.get("town", "") or location.get("village", "") or "la zona"

      # Construir system message
      system_content = (
          f"Eres un guía local experto y amigable. El usuario está caminando por {city} "
          f"y le interesan: {interests_str}.\n"
          "Tu objetivo es informar sobre lo que hay cerca, dar contexto histórico/cultural "
          "relevante a sus intereses, y sugerir qué visitar.\n"
          f"Responde en {lang}. Sé conciso (máximo 3-4 frases) y conversacional, "
          "como si caminaras junto al usuario.\n"
          "Si no hay POIs relevantes, comenta sobre el barrio o zona."
      )

      # Construir user message
      user_parts = []

      # Ubicación
      display_name = location.get("display_name", "")
      if display_name:
          user_parts.append(f"📍 Estoy en: {display_name}")

      neighbourhood = location.get("neighbourhood", "") or location.get("suburb", "")
      city_label = location.get("city", "") or location.get("town", "") or location.get("village", "")
      location_line_parts = [p for p in [neighbourhood, city_label] if p]
      if location_line_parts:
          user_parts.append(f"Barrio: {', '.join(location_line_parts)}")

      # POIs (máximo 5)
      top_pois = pois[:5]
      if top_pois:
          user_parts.append("")
          user_parts.append("Lugares cercanos que me pueden interesar:")
          for poi in top_pois:
              name = poi.get("name", "Sin nombre")
              category = poi.get("category", "")
              matched = poi.get("matched_interests", [])
              matched_str = ", ".join(matched) if matched else ""

              poi_line = f"- {name}"
              if category:
                  poi_line += f" ({category})"
              if matched_str:
                  poi_line += f" — relevante para: {matched_str}"
              user_parts.append(poi_line)

      # Query del usuario
      user_parts.append("")
      if user_query and user_query.strip():
          user_parts.append(user_query.strip())
      else:
          user_parts.append("¿Qué me puedes contar de esta zona?")

      user_content = "\n".join(user_parts)

      return [
          {"role": "system", "content": system_content},
          {"role": "user", "content": user_content},
      ]

params_schema:
  {"params":[{"name":"location","desc":"Dict con info de ubicación de nominatim_reverse_geocode. Campos esperados: display_name, street, neighbourhood, city, country. Los campos vacíos se omiten del prompt."},{"name":"pois","desc":"Lista de POIs rankeados por relevancia (ya filtrados por match_pois_to_interests). Cada POI tiene: name, category, score, matched_interests, lat, lon, tags. Se usan como máximo los primeros 5."},{"name":"interests","desc":"Lista de intereses del usuario. Cada interés tiene: name (str), keywords (list[str]), weight (float). Se usan los names para el system prompt."},{"name":"user_query","desc":"Pregunta opcional del usuario. Si está vacía o no se proporciona, se usa '¿Qué me puedes contar de esta zona?' como default."},{"name":"lang","desc":"Código de idioma para la respuesta del LLM (ej: 'es', 'en', 'fr'). Por defecto 'es'."}],"output":"Lista de dos mensajes en formato OpenAI/Ollama: [{'role': 'system', 'content': '...'}, {'role': 'user', 'content': '...'}]. El system message define el rol de guía local con los intereses y el idioma. El user message incluye la ubicación, los POIs relevantes y la query."}


id: build_tree_from_headers_py_core
name: build_tree_from_headers
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def build_tree_from_headers(node_list: list[dict]) -> list[dict]
description: Construye arbol jerarquico anidado desde lista plana de headers markdown con niveles (h1>h2>h3).
tags: ["tree","markdown","headers","hierarchy"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: bad37891f2f6e23a10e2928913e29b99183b0d26aa563647ae5e9d162926399c
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/page_index_md.py
example:
  ```python
  headers = [
      {"title": "Intro", "level": 1, "line_num": 1},
      {"title": "Background", "level": 2, "line_num": 5},
      {"title": "Details", "level": 3, "line_num": 10},
      {"title": "Methods", "level": 1, "line_num": 20},
  ]
  tree = build_tree_from_headers(headers)
  # [
  #   {"title": "Intro", "node_id": "0001", "nodes": [
  #     {"title": "Background", "node_id": "0002", "nodes": [
  #       {"title": "Details", "node_id": "0003"}
  #     ]}
  #   ]},
  #   {"title": "Methods", "node_id": "0004"}
  # ]
  ```

notes:
  Funcion pura. Asigna node_id secuencial (0001...) automaticamente. Usa stack para resolver jerarquia por nivel de header.

code:
  """Core functional programming utilities — pure functions for list/collection operations."""

  import hashlib
  import re
  from functools import reduce as _reduce
  from typing import Any, Callable, Dict, List, Optional, Tuple


  def filter_list(xs: list, pred: Callable) -> list:
      """Filter list by predicate. Does not mutate the original."""
      return [x for x in xs if pred(x)]


  def map_list(xs: list, fn: Callable) -> list:
      """Map function over list. Does not mutate the original."""
      return [fn(x) for x in xs]


  def reduce_list(xs: list, initial: Any, fn: Callable) -> Any:
      """Reduce list with accumulator. fn(acc, x) -> acc."""
      return _reduce(fn, xs, initial)


  def flat_map(xs: list, fn: Callable) -> list:
      """Map function over list then flatten one level."""
      result = []
      for x in xs:
          result.extend(fn(x))
      return result


  def flatten(xss: list) -> list:
      """Flatten a list of lists one level."""
      result = []
      for xs in xss:
          result.extend(xs)
      return result


  def chunk(xs: list, size: int) -> list:
      """Split list into chunks of given size. Last chunk may be smaller."""
      if size <= 0:
          return []
      return [xs[i : i + size] for i in range(0, len(xs), size)]


  def take(xs: list, n: int) -> list:
      """Take first n elements from list."""
      return xs[:n]


  def drop(xs: list, n: int) -> list:
      """Drop first n elements from list."""
      return xs[n:]


  def unique(xs: list) -> list:
      """Remove duplicates preserving order. Uses identity for hashable elements."""
      seen = set()
      result = []
      for x in xs:
          if x not in seen:
              seen.add(x)
              result.append(x)
      return result


  def group_by(xs: list, key_fn: Callable) -> Dict:
      """Group elements by key function. Returns dict of key -> list."""
      groups: Dict = {}
      for x in xs:
          k = key_fn(x)
          if k not in groups:
              groups[k] = []
          groups[k].append(x)
      return groups


  def partition(xs: list, pred: Callable) -> Tuple[list, list]:
      """Split list into (matches, non_matches) based on predicate."""
      matches = []
      non_matches = []
      for x in xs:
          if pred(x):
              matches.append(x)
          else:
              non_matches.append(x)
      return (matches, non_matches)


  def find(xs: list, pred: Callable) -> Any:
      """Find first element matching predicate. Returns None if not found."""
      for x in xs:
          if pred(x):
              return x
      return None


  def find_index(xs: list, pred: Callable) -> int:
      """Find index of first element matching predicate. Returns -1 if not found."""
      for i, x in enumerate(xs):
          if pred(x):
              return i
      return -1


  def zip_with(xs: list, ys: list, fn: Callable) -> list:
      """Zip two lists with a combining function. Stops at shorter list."""
      return [fn(x, y) for x, y in zip(xs, ys)]


  def all_of(xs: list, pred: Callable) -> bool:
      """Return True if all elements match predicate."""
      return all(pred(x) for x in xs)


  def any_of(xs: list, pred: Callable) -> bool:
      """Return True if any element matches predicate."""
      return any(pred(x) for x in xs)


  def pipe(value: Any, *fns: Callable) -> Any:
      """Pipe a value through a sequence of functions left-to-right."""
      result = value
      for fn in fns:
          result = fn(result)
      return result


  def compose(*fns: Callable) -> Callable:
      """Compose functions right-to-left. compose(f, g)(x) == f(g(x))."""
      def composed(x: Any) -> Any:
          result = x
          for fn in reversed(fns):
              result = fn(result)
          return result
      return composed


  # ── Tree manipulation ────────────────────────────────────────────────────────


  def flatten_tree(structure: Any) -> List[Dict]:
      """Flatten a hierarchical tree (dict with 'nodes') to a list without children."""
      import copy
      if isinstance(structure, dict):
          node = copy.deepcopy(structure)
          node.pop('nodes', None)
          nodes = [node]
          for key in list(structure.keys()):
              if 'nodes' in key:
                  nodes.extend(flatten_tree(structure[key]))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(flatten_tree(item))
          return nodes
      return []


  def tree_to_flat_list(structure: Any) -> List[Dict]:
      """Convert hierarchical tree to flat list preserving DFS order (keeps internal nodes)."""
      if isinstance(structure, dict):
          nodes = [structure]
          if 'nodes' in structure:
              nodes.extend(tree_to_flat_list(structure['nodes']))
          return nodes
      elif isinstance(structure, list):
          nodes = []
          for item in structure:
              nodes.extend(tree_to_flat_list(item))
          return nodes
      return []


  def get_leaf_nodes(structure: Any) -> List[Dict]:
      """Extract only leaf nodes (no children) from a hierarchical tree."""
      import copy
      if isinstance(structure, dict):
          if not structure.get('nodes'):
              node = copy.deepcopy(structure)
              node.pop('nodes', None)
              return [node]
          leaf_nodes = []
          for key in list(structure.keys()):
              if 'nodes' in key:
                  leaf_nodes.extend(get_leaf_nodes(structure[key]))
          return leaf_nodes
      elif isinstance(structure, list):
          leaf_nodes = []
          for item in structure:
              leaf_nodes.extend(get_leaf_nodes(item))
          return leaf_nodes
      return []


  def write_node_ids(data: Any, node_id: int = 0) -> int:
      """Assign sequential zero-padded IDs (0001, 0002...) to all nodes in a tree. Returns next counter."""
      if isinstance(data, dict):
          data['node_id'] = str(node_id).zfill(4)
          node_id += 1
          for key in list(data.keys()):
              if 'nodes' in key:
                  node_id = write_node_ids(data[key], node_id)
      elif isinstance(data, list):
          for item in data:
              node_id = write_node_ids(item, node_id)
      return node_id


  def list_to_tree(data: List[Dict]) -> List[Dict]:
      """Convert flat list with structure codes ('1.2.3') to nested tree."""
      def get_parent_structure(structure):
          if not structure:
              return None
          parts = str(structure).split('.')
          return '.'.join(parts[:-1]) if len(parts) > 1 else None

      nodes = {}
      root_nodes = []

      for item in data:
          structure = item.get('structure')
          node = {
              'title': item.get('title'),
              'start_index': item.get('start_index'),
              'end_index': item.get('end_index'),
              'nodes': []
          }
          nodes[structure] = node
          parent_structure = get_parent_structure(structure)

          if parent_structure and parent_structure in nodes:
              nodes[parent_structure]['nodes'].append(node)
          else:
              root_nodes.append(node)

      def clean_node(node):
          if not node['nodes']:
              del node['nodes']
          else:
              for child in node['nodes']:
                  clean_node(child)
          return node

      return [clean_node(node) for node in root_nodes]


  def remove_tree_fields(data: Any, fields: List[str] = None) -> Any:
      """Recursively remove specified fields from a tree (dict/list)."""
      if fields is None:
          fields = ['text']
      if isinstance(data, dict):
          return {k: remove_tree_fields(v, fields) for k, v in data.items() if k not in fields}
      elif isinstance(data, list):
          return [remove_tree_fields(item, fields) for item in data]
      return data


  def format_tree_structure(structure: Any, order: List[str] = None) -> Any:
      """Reorder fields of each node in a tree according to specified key order."""
      if not order:
          return structure
      if isinstance(structure, dict):
          if 'nodes' in structure:
              structure['nodes'] = format_tree_structure(structure['nodes'], order)
          if not structure.get('nodes'):
              structure.pop('nodes', None)
          return {key: structure[key] for key in order if key in structure}
      elif isinstance(structure, list):
          return [format_tree_structure(item, order) for item in structure]
      return structure


  def create_node_mapping(tree: List[Dict]) -> Dict[str, Dict]:
      """Create flat dict mapping node_id to node for O(1) lookup."""
      mapping = {}
      def _traverse(nodes):
          for node in nodes:
              if node.get('node_id'):
                  mapping[node['node_id']] = node
              if node.get('nodes'):
                  _traverse(node['nodes'])
      _traverse(tree)
      return mapping


  # ── Text / JSON extraction ───────────────────────────────────────────────────


  def extract_json_from_llm(content: str) -> Dict:
      """Extract and parse JSON from LLM responses. Handles ```json blocks, trailing commas, None->null."""
      import json
      try:
          start_idx = content.find("```json")
          if start_idx != -1:
              start_idx += 7
              end_idx = content.rfind("```")
              json_content = content[start_idx:end_idx].strip()
          else:
              json_content = content.strip()

          json_content = json_content.replace('None', 'null')
          json_content = json_content.replace('\n', ' ').replace('\r', ' ')
          json_content = ' '.join(json_content.split())

          return json.loads(json_content)
      except (json.JSONDecodeError, Exception):
          try:
              json_content = json_content.replace(',]', ']').replace(',}', '}')
              return json.loads(json_content)
          except Exception:
              return {}


  def parse_page_range(pages: str) -> List[int]:
      """Parse page range string ('5-7', '3,8', '12') into sorted list of unique ints."""
      result = []
      for part in pages.split(','):
          part = part.strip()
          if '-' in part:
              start, end = int(part.split('-', 1)[0].strip()), int(part.split('-', 1)[1].strip())
              if start > end:
                  raise ValueError(f"Invalid range '{part}': start must be <= end")
              result.extend(range(start, end + 1))
          else:
              result.append(int(part))
      return sorted(set(result))


  # ── Markdown parsing ─────────────────────────────────────────────────────────


  def extract_markdown_headers(markdown_content: str) -> Tuple[List[Dict], List[str]]:
      """Extract all headers (h1-h6) from markdown with line numbers, skipping code blocks."""
      import re
      header_pattern = r'^(#{1,6})\s+(.+)$'
      code_block_pattern = r'^```'
      node_list = []
      lines = markdown_content.split('\n')
      in_code_block = False

      for line_num, line in enumerate(lines, 1):
          stripped_line = line.strip()
          if re.match(code_block_pattern, stripped_line):
              in_code_block = not in_code_block
              continue
          if not stripped_line:
              continue
          if not in_code_block:
              match = re.match(header_pattern, stripped_line)
              if match:
                  level = len(match.group(1))
                  title = match.group(2).strip()
                  node_list.append({'title': title, 'level': level, 'line_num': line_num})

      return node_list, lines


  def build_tree_from_headers(node_list: List[Dict]) -> List[Dict]:
      """Build nested tree from flat list of headers with levels (h1>h2>h3)."""
      if not node_list:
          return []

      stack = []
      root_nodes = []
      node_counter = 1

      for node in node_list:
          current_level = node['level']
          tree_node = {
              'title': node['title'],
              'node_id': str(node_counter).zfill(4),
              'line_num': node['line_num'],
              'nodes': []
          }
          node_counter += 1

          while stack and stack[-1][1] >= current_level:
              stack.pop()

          if not stack:
              root_nodes.append(tree_node)
          else:
              parent_node, _ = stack[-1]
              parent_node['nodes'].append(tree_node)

          stack.append((tree_node, current_level))

      def clean_empty_nodes(nodes):
          for n in nodes:
              if n['nodes']:
                  clean_empty_nodes(n['nodes'])
              else:
                  del n['nodes']
          return nodes

      return clean_empty_nodes(root_nodes)


  # ── Pagination / chunking ────────────────────────────────────────────────────


  def page_list_to_groups(page_contents: List[str], token_lengths: List[int],
                          max_tokens: int = 20000, overlap_pages: int = 1) -> List[str]:
      """Group pages into text chunks respecting token limit with configurable overlap."""
      import math
      num_tokens = sum(token_lengths)

      if num_tokens <= max_tokens:
          return ["".join(page_contents)]

      subsets = []
      current_subset = []
      current_token_count = 0

      expected_parts = math.ceil(num_tokens / max_tokens)
      avg_tokens = math.ceil(((num_tokens / expected_parts) + max_tokens) / 2)

      for i, (page_content, page_tokens) in enumerate(zip(page_contents, token_lengths)):
          if current_token_count + page_tokens > avg_tokens:
              subsets.append(''.join(current_subset))
              overlap_start = max(i - overlap_pages, 0)
              current_subset = list(page_contents[overlap_start:i])
              current_token_count = sum(token_lengths[overlap_start:i])

          current_subset.append(page_content)
          current_token_count += page_tokens

      if current_subset:
          subsets.append(''.join(current_subset))

      return subsets


  def calculate_page_offset(pairs: List[Dict]) -> int:
      """Calculate offset between logical page numbers and physical indices using reference pairs."""
      differences = []
      for pair in pairs:
          try:
              difference = pair['physical_index'] - pair['page']
              differences.append(difference)
          except (KeyError, TypeError):
              continue

      if not differences:
          return 0

      counts: Dict[int, int] = {}
      for diff in differences:
          counts[diff] = counts.get(diff, 0) + 1

      return max(counts.items(), key=lambda x: x[1])[0]


  # ── Text preprocessing ───────────────────────────────────────────────────────


  def preprocess_text(text: str) -> str:
      """Normalize whitespace and newlines in raw text.

      Args:
          text: Raw text to normalize.

      Returns:
          Normalized text with consistent newlines, stripped lines, and no
          excessive blank lines.
      """
      # Normalize line endings: \r\n and \r -> \n
      text = text.replace('\r\n', '\n').replace('\r', '\n')
      # Reduce 3+ consecutive newlines to at most 2
      text = re.sub(r'\n{3,}', '\n\n', text)
      # Strip whitespace from each line
      text = '\n'.join(line.strip() for line in text.split('\n'))
      # Strip globally
      return text.strip()


  def get_text_stats(text: str) -> dict:
      """Compute basic statistics of a text: characters, lines, words.

      Args:
          text: Input text to analyze.

      Returns:
          Dict with keys total_chars (int), total_lines (int), total_words (int).
      """
      return {
          'total_chars': len(text),
          'total_lines': text.count('\n') + 1,
          'total_words': len(text.split()),
      }


  # ── Git URL parsing ──────────────────────────────────────────────────────────

  _DEFAULT_GIT_HOSTS = ["github.com", "gitlab.com"]


  def _sanitize_git_segment(segment: str) -> str:
      """Strip .git suffix then keep only [a-zA-Z0-9_-] chars."""
      if segment.endswith(".git"):
          segment = segment[:-4]
      return re.sub(r"[^a-zA-Z0-9_\-]", "", segment)


  def parse_git_url(url: str, known_hosts: Optional[List[str]] = None) -> Optional[str]:
      """Parse a code-hosting URL and return the 'org/repo' path component.

      Supports HTTPS, HTTP, git://, ssh:// and SSH shorthand (git@host:path).
      Returns None if the URL does not match any known host or is malformed.

      Args:
          url: Repository URL in any supported format.
          known_hosts: List of accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          'org/repo' string or None.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      if url.startswith("git@"):
          # git@github.com:org/repo.git
          rest = url[len("git@"):]
          if ":" not in rest:
              return None
          host, path = rest.split(":", 1)
          if host not in hosts:
              return None
          segments = [s for s in path.split("/") if s]
          if len(segments) < 2:
              return None
          org = _sanitize_git_segment(segments[0])
          repo = _sanitize_git_segment(segments[1])
          if not org or not repo:
              return None
          return f"{org}/{repo}"

      for prefix in ("http://", "https://", "git://", "ssh://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              netloc = parsed.hostname or ""
              if netloc not in hosts:
                  return None
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) < 2:
                  return None
              org = _sanitize_git_segment(segments[0])
              repo = _sanitize_git_segment(segments[1])
              if not org or not repo:
                  return None
              return f"{org}/{repo}"

      return None


  def is_git_repo_url(url: str, known_hosts: Optional[List[str]] = None) -> bool:
      """Return True only if url points to a clonable git repository.

      Accepts org/repo and org/repo/tree/<ref> paths.
      Rejects paths that navigate to sub-resources (issues, blobs, PRs, etc.).

      Args:
          url: URL to verify.
          known_hosts: Accepted hostnames. Defaults to github.com and gitlab.com.

      Returns:
          True if url is a clonable repository URL.
      """
      from urllib.parse import urlparse

      hosts = known_hosts if known_hosts is not None else _DEFAULT_GIT_HOSTS
      url = url.strip()

      # SSH shorthand — always repo-level if host matches
      if url.startswith("git@"):
          rest = url[len("git@"):]
          if ":" not in rest:
              return False
          host, _ = rest.split(":", 1)
          return host in hosts

      # git:// and ssh:// — always repo-level if host matches
      for prefix in ("ssh://", "git://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              return (parsed.hostname or "") in hosts

      # http:// and https:// — must have exactly org/repo or org/repo/tree/<ref>
      for prefix in ("http://", "https://"):
          if url.startswith(prefix):
              parsed = urlparse(url)
              if (parsed.hostname or "") not in hosts:
                  return False
              segments = [s for s in parsed.path.split("/") if s]
              if len(segments) == 2:
                  return True
              if len(segments) == 4 and segments[2] == "tree":
                  return True
              return False

      return False


  def validate_git_ssh_uri(url: str) -> None:
      """Validate a git SSH URI of the form git@host:path.

      Raises ValueError with a descriptive message if the URI is malformed.

      Args:
          url: URI string to validate.

      Raises:
          ValueError: If the URI does not conform to git SSH format.
      """
      if not url.startswith("git@"):
          raise ValueError(f"git SSH URI must start with 'git@', got: {url!r}")
      rest = url[len("git@"):]
      if ":" not in rest:
          raise ValueError(f"git SSH URI must contain ':', got: {url!r}")
      _, path = rest.split(":", 1)
      if not path:
          raise ValueError(f"git SSH URI must have a non-empty path after ':', got: {url!r}")


  # ---------------------------------------------------------------------------
  # Markdown parsing utilities
  # ---------------------------------------------------------------------------


  def extract_frontmatter(content: str) -> Tuple[str, Optional[Dict]]:
      """Extract YAML frontmatter delimited by '---' from the start of a markdown string.

      Args:
          content: Raw markdown string, optionally starting with YAML frontmatter.

      Returns:
          Tuple of (content_without_frontmatter, frontmatter_dict).
          frontmatter_dict is None when no frontmatter is found.
      """
      pattern = re.compile(r'^---\n(.*?)\n---\n', re.DOTALL)
      match = pattern.match(content)
      if not match:
          return content, None

      raw = match.group(1)
      remaining = content[match.end():]

      try:
          import yaml  # type: ignore
          data = yaml.safe_load(raw)
          if not isinstance(data, dict):
              data = None
      except Exception:
          # Fallback: simple key: value parser (no yaml dependency)
          data = {}
          for line in raw.splitlines():
              if ':' in line:
                  key, _, value = line.partition(':')
                  data[key.strip()] = value.strip()

      return remaining, data


  def find_headings(content: str) -> List[Tuple[int, int, str, int]]:
      """Find all markdown headings (# to ######), excluding those inside code blocks,
      HTML comments, and indented blocks.

      Args:
          content: Markdown text to search.

      Returns:
          List of (start_pos, end_pos, title, level) for each heading found.
      """
      excluded: List[Tuple[int, int]] = []

      # Code blocks (triple backtick)
      for m in re.finditer(r'```.*?```', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # HTML comments
      for m in re.finditer(r'<!--.*?-->', content, re.DOTALL):
          excluded.append((m.start(), m.end()))

      # Indented blocks (lines starting with 4 spaces or a tab)
      for m in re.finditer(r'^(    |\t).+$', content, re.MULTILINE):
          excluded.append((m.start(), m.end()))

      def is_excluded(pos: int) -> bool:
          return any(start <= pos < end for start, end in excluded)

      results: List[Tuple[int, int, str, int]] = []
      for m in re.finditer(r'^(#{1,6})\s+(.+)$', content, re.MULTILINE):
          # Skip escaped headings (\#)
          before = content[m.start() - 1] if m.start() > 0 else ''
          if before == '\\':
              continue
          if is_excluded(m.start()):
              continue
          level = len(m.group(1))
          title = m.group(2).strip()
          results.append((m.start(), m.end(), title, level))

      return results


  def estimate_token_count(content: str) -> int:
      """Estimate token count without a tokenizer.

      CJK characters count as ~0.7 tokens each; other non-whitespace characters
      count as ~0.3 tokens each.

      Args:
          content: Text to estimate.

      Returns:
          Estimated integer token count.
      """
      cjk = re.findall(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', content)
      without_cjk = re.sub(r'[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', '', content)
      others = re.findall(r'\S', without_cjk)
      return int(len(cjk) * 0.7 + len(others) * 0.3)


  def smart_split_content(
      content: str,
      max_tokens: int = 1024,
      max_chars: int = 8000,
  ) -> List[str]:
      """Split large content into parts respecting token and character limits.

      Splits by paragraphs (double newline). If a single paragraph exceeds the
      limit it is force-cut into chunks of max_chars.

      Args:
          content: Text to split.
          max_tokens: Maximum estimated tokens per part.
          max_chars: Maximum characters per part.

      Returns:
          List of string parts.
      """
      paragraphs = content.split('\n\n')
      parts: List[str] = []
      current_parts: List[str] = []
      current_tokens = 0
      current_chars = 0

      def flush() -> None:
          if current_parts:
              parts.append('\n\n'.join(current_parts))
              current_parts.clear()

      for para in paragraphs:
          para_tokens = estimate_token_count(para)
          para_chars = len(para)

          # Single paragraph exceeds limits — force-cut it
          if para_tokens > max_tokens or para_chars > max_chars:
              flush()
              current_tokens = 0
              current_chars = 0
              for i in range(0, len(para), max_chars):
                  parts.append(para[i:i + max_chars])
              continue

          # Would exceed limits if added — flush first
          if (current_tokens + para_tokens > max_tokens or
                  current_chars + para_chars > max_chars):
              flush()
              current_tokens = 0
              current_chars = 0

          current_parts.append(para)
          current_tokens += para_tokens
          current_chars += para_chars

      flush()
      return parts if parts else [content]


  def sanitize_for_path(text: str, max_length: int = 50) -> str:
      """Convert text to a safe string for use in file paths.

      Keeps word characters, CJK characters, spaces and hyphens. Replaces spaces
      with underscores. Truncates with a sha256 suffix if the result exceeds
      max_length.

      Args:
          text: Input text to sanitize.
          max_length: Maximum length of the returned string.

      Returns:
          Safe path-friendly string.
      """
      cleaned = re.sub(
          r'[^\w\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af \-]',
          '',
          text,
      )
      cleaned = cleaned.replace(' ', '_').strip('_')

      if not cleaned:
          return 'section'

      if len(cleaned) <= max_length:
          return cleaned

      suffix = '_' + hashlib.sha256(text.encode()).hexdigest()[:8]
      return cleaned[:max_length - len(suffix)] + suffix

params_schema:
  {"params":[{"name":"node_list","desc":"lista plana de dicts con campos title, level y opcionalmente line_num"}],"output":"lista de dicts que forman arbol jerarquico anidado segun niveles"}


id: cache_decorator_py_core
name: cache_decorator
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def cache_decorator(store: Any, ttl: float = 0, key_fn: callable | None = None)
tags: ["cache","decorator","memoize","persistence","async","functional"]
uses_functions: ["cache_to_sqlite_py_infra","cache_to_file_py_infra"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["asyncio","functools","hashlib","json"]
test_file_path: python/functions/core/cache_decorator_test.py
file_path: python/functions/core/cache_decorator.py
created_at: 2026-04-05T14:43:57Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: e3242f52e298ecb0650ab9a4ff464d2d94623051fb0ea3a3c693061e5fa57ac8
description:
  Decorator que cachea el resultado de una funcion en cualquier store persistente compatible (CacheStore o FileCache). La key se genera hasheando (func.__name__, args, sorted(kwargs)) con SHA-256. Soporta funciones sincronas y asincronas.

example:
  ```python
  from infra.cache_to_sqlite import cache_to_sqlite
  from core.cache_decorator import cache_decorator

  store = cache_to_sqlite("cache.db", namespace="llm")

  @cache_decorator(store, ttl=3600)
  def call_llm(prompt: str) -> str:
      # llamada costosa a LLM
      return client.complete(prompt)

  result = call_llm("explain X")  # primera vez: llama LLM
  result = call_llm("explain X")  # segunda vez: desde cache

  # Con key_fn custom
  @cache_decorator(store, ttl=600, key_fn=lambda fn, args, kw: args[0])
  def fetch_user(user_id: str) -> dict:
      return api.get_user(user_id)

  # Con async
  @cache_decorator(store, ttl=3600)
  async def async_call(prompt: str) -> str:
      return await async_client.complete(prompt)
  ```

tests:
  ["Funcion llamada una vez, segunda vez desde cache","TTL expirado → llama de nuevo","key_fn custom","Argumentos distintos → keys distintas","Funciona con async"]

notes:
  El store debe implementar `get(key: str) -> Any | None` y `set(key: str, value: Any, ttl: float) -> None`. Detecta automaticamente funciones asincronas con `asyncio.iscoroutinefunction`. La key por defecto usa `json.dumps(..., default=str)` para serializar argumentos no serializables. Si `store.get()` retorna `None`, siempre se ejecuta la funcion (no distingue entre "no en cache" y "valor None almacenado"); para valores que pueden ser None usar `get_or_set` directamente.

code:
  """Decorator que cachea el resultado de una funcion en un store persistente."""

  import asyncio
  import functools
  import hashlib
  import json
  from typing import Any, Callable


  def _default_key(func: Callable, args: tuple, kwargs: dict) -> str:
      """Genera una cache key a partir del nombre de funcion y sus argumentos."""
      payload = json.dumps((func.__name__, args, sorted(kwargs.items())), default=str)
      return hashlib.sha256(payload.encode("utf-8")).hexdigest()


  def cache_decorator(store: Any, ttl: float = 0, key_fn: Callable | None = None):
      """Retorna un decorator que cachea resultados en un store persistente.

      Args:
          store: Cualquier objeto con metodos get(key) y set(key, value, ttl).
                 Compatible con CacheStore (cache_to_sqlite) y FileCache (cache_to_file).
          ttl: Tiempo de vida en segundos. 0 = sin expiracion.
          key_fn: Funcion opcional para generar la key. Recibe (func, args, kwargs).
                  Si es None, se usa SHA-256 de (func.__name__, args, sorted(kwargs)).

      Returns:
          Decorator aplicable a funciones sincronas o asincronas.

      Example::

          store = cache_to_sqlite("cache.db")

          @cache_decorator(store, ttl=3600)
          def call_llm(prompt: str) -> str:
              ...  # llamada costosa

          result = call_llm("explain X")  # primera vez: ejecuta la funcion
          result = call_llm("explain X")  # segunda vez: desde cache
      """

      def decorator(func: Callable) -> Callable:
          if asyncio.iscoroutinefunction(func):
              @functools.wraps(func)
              async def async_wrapper(*args, **kwargs):
                  make_key = key_fn or _default_key
                  key = make_key(func, args, kwargs)
                  cached = store.get(key)
                  if cached is not None:
                      return cached
                  result = await func(*args, **kwargs)
                  store.set(key, result, ttl)
                  return result
              return async_wrapper
          else:
              @functools.wraps(func)
              def sync_wrapper(*args, **kwargs):
                  make_key = key_fn or _default_key
                  key = make_key(func, args, kwargs)
                  cached = store.get(key)
                  if cached is not None:
                      return cached
                  result = func(*args, **kwargs)
                  store.set(key, result, ttl)
                  return result
              return sync_wrapper

      return decorator

params_schema:
  {"params":[{"name":"store","desc":"store persistente compatible (CacheStore o FileCache) que implementa get/set"},{"name":"ttl","desc":"tiempo de vida del cache en segundos (0 significa infinito)"},{"name":"key_fn","desc":"funcion personalizada para generar clave de cache, recibe (func_name, args, kwargs)"}],"output":"decorator que cachea resultados de la funcion decorada"}


id: calculate_media_strategy_py_core
name: calculate_media_strategy
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: calculate_media_strategy(image_count: int, line_count: int) -> str
tags: ["media","strategy","document","vision","vlm","images","classification"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/calculate_media_strategy_test.py
file_path: python/functions/core/calculate_media_strategy.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 31a5107b56d4f03059c9011c13953059f999437a56f5299661a83cf64046e876
description:
  Determina la estrategia optima de procesamiento de medios para un documento basado en la proporcion de imagenes vs texto. Retorna full_page_vlm, extract o text_only.

example:
  ```python
  calculate_media_strategy(0, 50)    # "text_only"
  calculate_media_strategy(2, 100)   # "extract"  (ratio 0.02, pocas imagenes)
  calculate_media_strategy(10, 20)   # "full_page_vlm"  (ratio 0.5 > 0.3)
  calculate_media_strategy(5, 100)   # "full_page_vlm"  (>= 5 imagenes)
  calculate_media_strategy(3, 0)     # "text_only"  (sin texto, sin contexto)
  ```

tests:
  ["0 imagenes text_only","2 imagenes 100 lineas extract","10 imagenes 20 lineas full_page_vlm","5 imagenes 100 lineas full_page_vlm","0 lineas division por cero evitada"]

notes:
  Logica de clasificacion en tres niveles:

  1. `full_page_vlm` — documento dominado por imagenes: ratio imagen/linea > 0.3 o al menos 5 imagenes. Se usa un vision-language model sobre la pagina completa.
  2. `extract` — pocas imagenes en documento con texto: extraer y procesar imagenes individualmente.
  3. `text_only` — sin imagenes o sin lineas de texto: procesar solo el texto.

  El guard `line_count > 0` evita la division por cero y trata documentos sin lineas como `text_only` independientemente del conteo de imagenes, ya que sin texto no hay contexto suficiente para clasificar como `extract`.

  Funcion pura, sin dependencias externas. Reimplementada conceptualmente a partir de la logica de clasificacion de medios de OpenViking (AGPL-3.0).

code:
  """Determina la estrategia optima de procesamiento de medios para un documento."""


  def calculate_media_strategy(image_count: int, line_count: int) -> str:
      """Determina la estrategia optima de procesamiento de medios.

      Clasifica un documento en una de tres estrategias basandose en la
      proporcion de imagenes respecto al texto:
      - full_page_vlm: documento dominado por imagenes, usar vision-language model
      - extract: pocas imagenes, extraer y procesar individualmente
      - text_only: sin imagenes, solo texto

      Args:
          image_count: numero de imagenes en el documento.
          line_count: numero de lineas de texto en el documento.

      Returns:
          "full_page_vlm", "extract" o "text_only".
      """
      if line_count > 0 and (image_count / line_count > 0.3 or image_count >= 5):
          return "full_page_vlm"
      if line_count > 0 and image_count > 0:
          return "extract"
      return "text_only"

params_schema:
  {"params":[{"name":"image_count","desc":"numero total de imagenes en el documento"},{"name":"line_count","desc":"numero total de lineas de texto en el documento"}],"output":"string con estrategia optima: 'full_page_vlm', 'extract' o 'text_only'"}


id: calculate_page_offset_py_core
name: calculate_page_offset
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def calculate_page_offset(pairs: list[dict]) -> int
tags: ["pagination","offset","calculation"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 26657b069c7a9c810fd4b978375eae3d88fccb7eecc6d1fda9a285cf7144c8d5
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/page_index.py
description:
  Calcula offset entre numeros de pagina logicos y fisicos usando pares de referencia (moda de diferencias).

example:
  ```python
  pairs = [
      {"page": 1, "physical_index": 5},
      {"page": 2, "physical_index": 6},
      {"page": 10, "physical_index": 14},
  ]
  calculate_page_offset(pairs)
  # 4 (la moda de las diferencias physical_index - page)
  ```

notes:
  Funcion pura. Cada par necesita campos 'page' (numero logico) y 'physical_index' (indice fisico). Retorna la diferencia mas frecuente (moda). Retorna 0 si no hay pares validos.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"pairs","desc":"lista de dicts con campos 'page' (numero logico) y 'physical_index' (indice fisico)"}],"output":"entero con el offset (diferencia moda entre physical_index - page)"}


id: call_batch_with_retry_py_core
name: call_batch_with_retry
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["retry","batch","backoff","resilience","error-handling","core"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["time","random","typing.Callable","typing.TypeVar"]
test_file_path: python/functions/core/call_batch_with_retry_test.py
file_path: python/functions/core/call_batch_with_retry.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 0ce4898f428d0b7a2f9c87ff0edb4ae7b83cb8025e736a94789b0d5a04c4392b
signature:
  def call_batch_with_retry(items: list[T], process_func: Callable[[T], R], max_retries: int = 3, initial_delay: float = 1.0, max_delay: float = 30.0, backoff_factor: float = 2.0, exceptions: tuple[type[Exception], ...] = (Exception,), continue_on_failure: bool = True) -> tuple[list[R], list[dict]]

description:
  Procesa una lista de items con retry individual por item y exponential backoff. Los fallos individuales no bloquean el resto del batch. Retorna (results, failures) donde failures contiene index, item y error de cada item que agoto sus reintentos.

example:
  ```python
  results, failures = call_batch_with_retry(
      items=["url1", "url2", "url3"],
      process_func=fetch_url,
      max_retries=3,
      initial_delay=1.0,
      max_delay=30.0,
      backoff_factor=2.0,
      exceptions=(ConnectionError, TimeoutError),
      continue_on_failure=True,
  )

  for r in results:
      print("OK:", r)

  for f in failures:
      print(f"FAIL index={f['index']} item={f['item']} error={f['error']}")
  ```

tests:
  ["todos los items exito","item falla permanentemente, continue True","item falla, abort continue False","item falla luego exito retry funciona","failures contiene index correcto"]

notes:
  Diferencia con `retry_sync_py_core`: ese reintenta una sola llamada. Este maneja listas completas donde cada item se reintenta independientemente — los fallos individuales quedan registrados en `failures` sin interrumpir el procesamiento del batch (cuando `continue_on_failure=True`).

  El backoff usa la formula `min(initial_delay * backoff_factor^attempt, max_delay)` con jitter de hasta el 10% del delay calculado para evitar thundering herd. El primer intento es siempre inmediato — el delay se aplica antes del primer retry (attempt=0).

  Cuando `continue_on_failure=False`, el primer item que agota sus reintentos re-lanza la excepcion inmediatamente, abortando el batch.

code:
  """Process a batch of items with per-item exponential backoff retry."""

  import time
  import random
  from typing import Callable, TypeVar

  T = TypeVar("T")
  R = TypeVar("R")


  def call_batch_with_retry(
      items: list,
      process_func: Callable,
      max_retries: int = 3,
      initial_delay: float = 1.0,
      max_delay: float = 30.0,
      backoff_factor: float = 2.0,
      exceptions: tuple = (Exception,),
      continue_on_failure: bool = True,
  ) -> tuple:
      """Process a list of items with independent per-item retry and exponential backoff.

      Each item is processed by process_func. If it raises one of the specified
      exceptions, it is retried up to max_retries times with exponential backoff.
      If all retries are exhausted, the item is recorded as a failure.

      Args:
          items: List of items to process.
          process_func: Callable that takes a single item and returns a result.
          max_retries: Maximum number of retry attempts per item after first failure.
          initial_delay: Initial delay in seconds before the first retry.
          max_delay: Maximum delay cap in seconds between retries.
          backoff_factor: Multiplier applied to delay on each successive retry.
          exceptions: Tuple of exception types to catch and retry on.
          continue_on_failure: If True, continue processing remaining items when an
              item exhausts all retries. If False, re-raise the exception immediately.

      Returns:
          A tuple (results, failures) where:
          - results is a list of successful return values from process_func.
          - failures is a list of dicts with keys "index", "item", and "error"
            for each item that failed after all retries.

      Raises:
          Exception: The last exception for a failed item when continue_on_failure
              is False.
      """
      results = []
      failures = []

      for index, item in enumerate(items):
          last_exc = None
          succeeded = False

          for attempt in range(max_retries + 1):
              try:
                  result = process_func(item)
                  results.append(result)
                  succeeded = True
                  break
              except exceptions as exc:
                  last_exc = exc
                  if attempt < max_retries:
                      delay = min(
                          initial_delay * (backoff_factor ** attempt),
                          max_delay,
                      )
                      # Add small jitter (up to 10% of delay) to avoid thundering herd
                      delay += random.uniform(0, delay * 0.1)
                      time.sleep(delay)

          if not succeeded:
              if not continue_on_failure:
                  raise last_exc
              failures.append({
                  "index": index,
                  "item": item,
                  "error": str(last_exc),
              })

      return results, failures

params_schema:
  {"params":[{"name":"items","desc":"lista de items a procesar"},{"name":"process_func","desc":"funcion que procesa un item individual"},{"name":"max_retries","desc":"numero maximo de reintentos por item"},{"name":"initial_delay","desc":"delay inicial en segundos para exponential backoff"},{"name":"max_delay","desc":"delay maximo en segundos para el backoff"},{"name":"backoff_factor","desc":"factor de multiplicacion para el backoff exponencial"},{"name":"exceptions","desc":"tupla de tipos de excepcion que desencadenan reintentos"},{"name":"continue_on_failure","desc":"si False, aborta el batch en el primer error; si True continua"}],"output":"tupla (results, failures) donde results es lista de valores y failures es lista de dicts con index/item/error"}


id: chunk_py_core
name: chunk
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def chunk(xs: list, size: int) -> list
description: Divide una lista en sublistas de tamanio fijo. El ultimo chunk puede ser menor.
tags: ["list","functional","chunk","partition","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = chunk([1, 2, 3, 4, 5], 2)
  # [[1, 2], [3, 4], [5]]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Si size <= 0 retorna lista vacia.
content_hash: 6d9e89836a7b834ab3e07190ca55ee22912a2af04a3575ff40b990f3e7bbd757
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista a dividir"},{"name":"size","desc":"tamanio de cada sublista (chunk)"}],"output":"lista de sublistas, cada una con maximo 'size' elementos"}


id: circuit_breaker_py_core
name: circuit_breaker
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["circuit-breaker","resilience","api","retry","error-handling","thread-safe"]
uses_functions: ["classify_api_error_py_core"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["threading","time","enum"]
test_file_path: python/functions/core/circuit_breaker_test.py
file_path: python/functions/core/circuit_breaker.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 5ace17a12c63da75881b967c68c31c5bcb600111ad6489a4de46ea8550abe128
signature:
  class CircuitBreaker:
    def __init__(self, failure_threshold: int = 5, reset_timeout: float = 300.0): ...
    def check(self) -> None: ...
    def record_success(self) -> None: ...
    def record_failure(self, error: Exception) -> None: ...
    @property
    def retry_after(self) -> float: ...

description:
  Patron circuit breaker thread-safe para proteger llamadas a APIs externas. Tres estados: CLOSED (normal), OPEN (bloqueando), HALF_OPEN (permitiendo 1 request de prueba). Integra con classify_api_error para distinguir errores permanentes de transitorios.

example:
  ```python
  from circuit_breaker import CircuitBreaker, CircuitBreakerOpen

  cb = CircuitBreaker(failure_threshold=3, reset_timeout=60.0)

  def call_api() -> dict:
      cb.check()  # raises CircuitBreakerOpen if circuit is open
      try:
          result = requests.get("https://api.example.com/data").json()
          cb.record_success()
          return result
      except Exception as exc:
          cb.record_failure(exc)
          raise

  # After 3 consecutive failures the circuit opens:
  # CircuitBreakerOpen: Circuit breaker is open. Retry after 30.0s
  try:
      cb.check()
  except CircuitBreakerOpen as e:
      print(f"Circuit open, retry in {e.retry_after}s")

  # retry_after property (capped at 30s):
  print(cb.retry_after)  # e.g. 28.4
  ```

tests:
  ["Transicion CLOSED → OPEN despues de N fallos","Transicion OPEN → HALF_OPEN despues de timeout","Transicion HALF_OPEN → CLOSED en exito","Transicion HALF_OPEN → OPEN en fallo","Error permanente abre inmediatamente","Thread safety (concurrencia)","retry_after retorna 0 cuando no esta OPEN"]

notes:
  - **CLOSED**: Requests pasan normalmente. Tras `failure_threshold` fallos consecutivos transiciona a OPEN.
  - **OPEN**: Requests bloqueados con `CircuitBreakerOpen`. Tras `reset_timeout` segundos transiciona a HALF_OPEN.
  - **HALF_OPEN**: Permite 1 request de prueba. Exito → CLOSED. Fallo → OPEN.
  - Errores permanentes (401, 403) abren el circuito inmediatamente sin esperar al umbral.
  - `retry_after` devuelve 0.0 cuando el estado no es OPEN; en OPEN devuelve el tiempo restante, cap 30s.
  - Thread-safe via `threading.Lock` protegiendo todo el estado interno.
  - La dependencia en `classify_api_error` es opcional: si no se puede importar, hay fallback de texto.

code:
  """Circuit breaker pattern for protecting external API calls."""

  import threading
  import time
  from enum import Enum


  class CircuitBreakerState(Enum):
      CLOSED = "closed"
      OPEN = "open"
      HALF_OPEN = "half_open"


  class CircuitBreakerOpen(Exception):
      """Raised when the circuit breaker is open and blocking requests."""

      def __init__(self, retry_after: float) -> None:
          self.retry_after = retry_after
          super().__init__(f"Circuit breaker is open. Retry after {retry_after:.1f}s")


  def _is_permanent_error(error: Exception) -> bool:
      """Return True if the error is permanent (should open circuit immediately)."""
      try:
          from classify_api_error import classify_api_error

          return classify_api_error(error) == "permanent"
      except ImportError:
          # Fallback: inspect error text directly
          text = str(error)
          if error.__cause__ is not None:
              text += " " + str(error.__cause__)
          permanent_patterns = ["400", "401", "403", "Forbidden", "Unauthorized"]
          return any(p in text for p in permanent_patterns)


  class CircuitBreaker:
      """Thread-safe circuit breaker for protecting external API calls.

      Implements three states:
      - CLOSED: requests pass through normally.
      - OPEN: requests are blocked with CircuitBreakerOpen.
      - HALF_OPEN: one probe request is allowed through.

      Args:
          failure_threshold: Consecutive failures before opening. Default 5.
          reset_timeout: Seconds to wait in OPEN before trying HALF_OPEN. Default 300.0.
      """

      def __init__(
          self,
          failure_threshold: int = 5,
          reset_timeout: float = 300.0,
      ) -> None:
          self._failure_threshold = failure_threshold
          self._reset_timeout = reset_timeout
          self._lock = threading.Lock()

          self._state = CircuitBreakerState.CLOSED
          self._failure_count = 0
          self._opened_at: float | None = None

      # ------------------------------------------------------------------
      # Public interface
      # ------------------------------------------------------------------

      def check(self) -> None:
          """Check whether a request is allowed through.

          Raises:
              CircuitBreakerOpen: If the circuit is open and reset_timeout
                  has not elapsed yet.
          """
          with self._lock:
              if self._state is CircuitBreakerState.CLOSED:
                  return

              if self._state is CircuitBreakerState.OPEN:
                  elapsed = time.monotonic() - self._opened_at  # type: ignore[operator]
                  if elapsed >= self._reset_timeout:
                      self._state = CircuitBreakerState.HALF_OPEN
                      return
                  remaining = self._reset_timeout - elapsed
                  raise CircuitBreakerOpen(min(remaining, 30.0))

              # HALF_OPEN: allow exactly one probe — caller holds the slot
              if self._state is CircuitBreakerState.HALF_OPEN:
                  return

      def record_success(self) -> None:
          """Record a successful request. Resets the breaker to CLOSED."""
          with self._lock:
              self._state = CircuitBreakerState.CLOSED
              self._failure_count = 0
              self._opened_at = None

      def record_failure(self, error: Exception) -> None:
          """Record a failed request.

          If the error is permanent (e.g. 401/403), opens immediately.
          Otherwise increments the failure counter and opens once it
          reaches failure_threshold.

          Args:
              error: The exception that was raised.
          """
          with self._lock:
              if _is_permanent_error(error):
                  self._trip()
                  return

              if self._state is CircuitBreakerState.HALF_OPEN:
                  self._trip()
                  return

              self._failure_count += 1
              if self._failure_count >= self._failure_threshold:
                  self._trip()

      @property
      def retry_after(self) -> float:
          """Seconds until the circuit transitions to HALF_OPEN.

          Returns 0.0 when not in OPEN state, capped at 30 seconds.
          """
          with self._lock:
              if self._state is not CircuitBreakerState.OPEN:
                  return 0.0
              elapsed = time.monotonic() - self._opened_at  # type: ignore[operator]
              remaining = self._reset_timeout - elapsed
              return min(max(remaining, 0.0), 30.0)

      # ------------------------------------------------------------------
      # Internal helpers
      # ------------------------------------------------------------------

      def _trip(self) -> None:
          """Open the circuit (must be called with _lock held)."""
          self._state = CircuitBreakerState.OPEN
          self._failure_count = 0
          self._opened_at = time.monotonic()

params_schema:
  {"params":[{"name":"failure_threshold","desc":"numero de fallos consecutivos que dispara el estado OPEN"},{"name":"reset_timeout","desc":"tiempo en segundos que debe esperar antes de pasar a HALF_OPEN"}],"output":"instancia de CircuitBreaker con metodos check(), record_success(), record_failure(), retry_after"}


id: classify_api_error_py_core
name: classify_api_error
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def classify_api_error(error: Exception) -> str
tags: ["retry","error","classification","api","backoff"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/classify_api_error_test.py
file_path: python/functions/core/classify_api_error.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: acebfbd4a4bbc25388247fa98202ff5f10dced1bb6f46420984ce8b7c03006fd
description:
  Clasifica un error de API como permanente (no reintentar), transitorio (reintentar) o desconocido. Permanente tiene prioridad sobre transitorio.

example:
  ```python
  err = Exception("HTTP 429 TooManyRequests")
  classify_api_error(err)  # "transient"

  err = Exception("HTTP 401 Unauthorized")
  classify_api_error(err)  # "permanent"

  err = Exception("Connection timeout")
  classify_api_error(err)  # "transient"

  err = Exception("Something unexpected happened")
  classify_api_error(err)  # "unknown"
  ```

tests:
  ["error 429 es transitorio","error 401 es permanente","error timeout es transitorio","error desconocido retorna unknown","error con __cause__ transitorio"]

notes:
  Funcion pura: solo inspecciona el texto del error y su causa directa (`__cause__`). No tiene I/O ni dependencias externas. La prioridad permanente > transitorio evita reintentar errores 400/401/403 que nunca tendran exito.

code:
  """Classify an API exception as permanent, transient, or unknown."""


  def classify_api_error(error: Exception) -> str:
      """Classify an API error as permanent, transient, or unknown.

      Permanent errors should not be retried (e.g. auth failures, bad requests).
      Transient errors are safe to retry (e.g. rate limits, timeouts, server errors).
      Permanent classification takes priority over transient.

      Args:
          error: The exception to classify.

      Returns:
          "permanent" | "transient" | "unknown"
      """
      parts = [str(error)]
      if error.__cause__ is not None:
          parts.append(str(error.__cause__))
      text = " ".join(parts)

      permanent_patterns = ["400", "401", "403", "Forbidden", "Unauthorized"]
      transient_patterns = [
          "429", "500", "502", "503", "504",
          "TooManyRequests", "RateLimit",
          "timeout", "Timeout",
          "ConnectionError", "Connection refused", "Connection reset",
      ]

      for pattern in permanent_patterns:
          if pattern in text:
              return "permanent"

      for pattern in transient_patterns:
          if pattern in text:
              return "transient"

      return "unknown"

params_schema:
  {"params":[{"name":"error","desc":"excepcion de Python (Exception o subclase) a clasificar"}],"output":"string: 'permanent' (no reintentar), 'transient' (reintentar), o 'unknown'"}


id: coerce_types_py_core
name: coerce_types
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def coerce_types(data: dict, schema: dict[str, str]) -> tuple[dict, list[str]]
tags: ["coercion","types","normalization","pure","core","csv","json"]
uses_functions: []
uses_types: []
returns: []
imports: ["datetime"]
test_file_path: python/functions/core/coerce_types_test.py
file_path: python/functions/core/coerce_types.py
created_at: 2026-04-05T11:24:00Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 08b428f780fe94f7ac5ff6d00439be2883acb13abbdbb92a063aa034f03affee
description:
  Convierte valores de un dict a los tipos esperados segun un schema declarativo. Soporta int, float, str, bool, datetime, list[str]. Util para normalizar datos de CSV, JSON o query params. Nunca muta el original. Coerciones imposibles generan warning y mantienen el valor original.

example:
  ```python
  data = {"age": "25", "score": "9.5", "active": "yes", "tags": "go, python"}
  schema = {"age": "int", "score": "float", "active": "bool", "tags": "list[str]"}

  result, warnings = coerce_types(data, schema)
  # result = {"age": 25, "score": 9.5, "active": True, "tags": ["go", "python"]}
  # warnings = []

  # Coercion fallida — mantiene original y avisa
  result2, warnings2 = coerce_types({"n": "abc"}, {"n": "int"})
  # result2 = {"n": "abc"}
  # warnings2 = ["n: cannot coerce 'abc' to int: could not convert string to float: 'abc'"]
  ```

tests:
  ["string 42 a int 42","string 3.14 a float 3.14","string true a bool true","string iso8601 a datetime","coercion fallida genera warning sin crash","dict con mix de tipos ya correctos y strings","campo ausente en schema pass through sin tocar","string lista a list str"]

notes:
  Funcion pura. Solo usa `datetime` de la stdlib. No muta el dict original — retorna uno nuevo. Schema es flat (no anidado); para validacion de estructura compleja combinar con `validate_json_schema`. Lossy coercions (float "3.7" → int 3) generan warning adicional. Campo ausente en schema se copia sin tocar.

code:
  """Coercion de valores de un dict a tipos esperados segun un schema declarativo."""

  from datetime import datetime, timezone


  def coerce_types(
      data: dict, schema: dict[str, str]
  ) -> tuple[dict, list[str]]:
      """Convierte valores de un dict a los tipos esperados segun el schema.

      Schema es un dict de {campo: tipo} donde tipo es uno de:
      "int", "float", "str", "bool", "datetime", "list[str]".

      Coerciones soportadas (todas desde str):
      - str → int: int(v), warning si tenia decimales
      - str → float: float(v)
      - str → bool: "true/1/yes" → True, "false/0/no" → False (case-insensitive)
      - str → datetime: ISO 8601 parse
      - str → list[str]: split por "," y strip de cada elemento
      - Valor ya del tipo correcto → pass through
      - Campo ausente en schema → pass through sin tocar
      - Coercion imposible → mantener original + warning

      Args:
          data: Dict con los valores a coercionar.
          schema: Dict de {campo: tipo_esperado}.

      Returns:
          (coerced_data, warnings) — nuevo dict con tipos corregidos (no muta el
          original), lista de warnings para coerciones lossy o fallidas.
      """
      result = dict(data)
      warnings: list[str] = []

      for field, target_type in schema.items():
          if field not in data:
              continue

          value = data[field]
          try:
              result[field] = _coerce_value(value, target_type, field, warnings)
          except Exception as exc:
              warnings.append(
                  f"{field}: cannot coerce {value!r} to {target_type}: {exc}"
              )
              result[field] = value

      return result, warnings


  _BOOL_TRUE = {"true", "1", "yes"}
  _BOOL_FALSE = {"false", "0", "no"}


  def _coerce_value(
      value: object, target: str, field: str, warnings: list[str]
  ) -> object:
      # --- int ---
      if target == "int":
          if isinstance(value, int) and not isinstance(value, bool):
              return value
          if isinstance(value, float):
              if value != int(value):
                  warnings.append(
                      f"{field}: lossy coercion float→int: {value} → {int(value)}"
                  )
              return int(value)
          if isinstance(value, str):
              stripped = value.strip()
              # detectar si tiene parte decimal no cero
              try:
                  as_float = float(stripped)
                  if as_float != int(as_float):
                      warnings.append(
                          f"{field}: lossy coercion str→int: {value!r} → {int(as_float)}"
                      )
                  return int(as_float)
              except ValueError:
                  raise ValueError(f"cannot parse {value!r} as int")
          raise TypeError(f"cannot coerce {type(value).__name__} to int")

      # --- float ---
      if target == "float":
          if isinstance(value, float):
              return value
          if isinstance(value, int) and not isinstance(value, bool):
              return float(value)
          if isinstance(value, str):
              return float(value.strip())
          raise TypeError(f"cannot coerce {type(value).__name__} to float")

      # --- str ---
      if target == "str":
          if isinstance(value, str):
              return value
          return str(value)

      # --- bool ---
      if target == "bool":
          if isinstance(value, bool):
              return value
          if isinstance(value, str):
              low = value.strip().lower()
              if low in _BOOL_TRUE:
                  return True
              if low in _BOOL_FALSE:
                  return False
              raise ValueError(
                  f"cannot parse {value!r} as bool; expected true/false/1/0/yes/no"
              )
          if isinstance(value, int):
              return bool(value)
          raise TypeError(f"cannot coerce {type(value).__name__} to bool")

      # --- datetime ---
      if target == "datetime":
          if isinstance(value, datetime):
              return value
          if isinstance(value, str):
              s = value.strip()
              # Intentar parse ISO 8601 con y sin Z
              if s.endswith("Z"):
                  s = s[:-1] + "+00:00"
              return datetime.fromisoformat(s)
          raise TypeError(f"cannot coerce {type(value).__name__} to datetime")

      # --- list[str] ---
      if target == "list[str]":
          if isinstance(value, list):
              return [str(item) for item in value]
          if isinstance(value, str):
              return [item.strip() for item in value.split(",")]
          raise TypeError(f"cannot coerce {type(value).__name__} to list[str]")

      raise ValueError(f"unknown target type: {target!r}")

params_schema:
  {"params":[{"name":"data","desc":"dict con valores a coercionar (tipicamente strings)"},{"name":"schema","desc":"dict con campos como keys y tipos esperados como values (int, float, str, bool, datetime, list[str])"}],"output":"tupla (resultado_coercionado, lista_de_warnings)"}


id: compose_py_core
name: compose
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def compose(*fns) -> callable
description: Compone funciones de derecha a izquierda. compose(f, g)(x) == f(g(x)).
tags: ["functional","compose","composition","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  double_then_str = compose(str, lambda n: n * 2)
  result = double_then_str(5)
  # "10"
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Composicion matematica: la ultima funcion se aplica primero. Inverso de pipe.
content_hash: 8f13520b4235704d3dc92a28e7382863cba13791fac664802c3c27a83cef6c3b
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"fns","desc":"funciones a componer (variable number of arguments)"}],"output":"funcion compuesta que aplica las funciones de derecha a izquierda"}


id: compute_backoff_delay_py_core
name: compute_backoff_delay
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["retry","backoff","exponential","delay","jitter"]
uses_functions: []
uses_types: []
returns: []
imports: ["random"]
test_file_path: python/functions/core/compute_backoff_delay_test.py
file_path: python/functions/core/compute_backoff_delay.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: feb4750b115658d30eabebd377d1ab61c4bf1ab93eaa2f61047be6fdfe826afe
signature:
  def compute_backoff_delay(attempt: int, base_delay: float = 0.5, max_delay: float = 8.0, jitter: bool = True) -> float

description:
  Calcula el delay para exponential backoff con jitter opcional. delay = min(base_delay * 2^attempt, max_delay). Con jitter anade random.uniform(0, min(base_delay, delay)).

example:
  ```python
  # Primer reintento (attempt=0): delay = 0.5 * 2^0 = 0.5s
  compute_backoff_delay(0, jitter=False)  # 0.5

  # Tercer reintento (attempt=2): delay = 0.5 * 2^2 = 2.0s
  compute_backoff_delay(2, jitter=False)  # 2.0

  # Intento alto, capped a 8.0s
  compute_backoff_delay(10, jitter=False)  # 8.0

  # Con jitter (no determinista)
  compute_backoff_delay(1)  # entre 1.0 y 1.5
  ```

tests:
  ["attempt 0 retorna base_delay sin jitter","attempt alto se cappea a max_delay","sin jitter es determinista"]

notes:
  Usa `random` de la stdlib. Con jitter=True el resultado no es determinista, pero la funcion es clasificada como pura conceptualmente dado que el jitter es intencional y no hay I/O. Para tests deterministicos usar jitter=False.

code:
  """Compute exponential backoff delay with optional jitter."""

  import random


  def compute_backoff_delay(
      attempt: int,
      base_delay: float = 0.5,
      max_delay: float = 8.0,
      jitter: bool = True,
  ) -> float:
      """Compute exponential backoff delay for a given attempt number.

      Args:
          attempt: Zero-based attempt index (0 = first retry).
          base_delay: Base delay in seconds before exponential scaling.
          max_delay: Maximum delay cap in seconds.
          jitter: If True, adds random jitter to avoid thundering herd.

      Returns:
          Delay in seconds to wait before the next attempt.
      """
      delay = min(base_delay * (2 ** attempt), max_delay)
      if jitter:
          delay += random.uniform(0, min(base_delay, delay))
      return delay

params_schema:
  {"params":[{"name":"attempt","desc":"numero de intento (0-based, siendo 0 el primer reintento)"},{"name":"base_delay","desc":"delay inicial en segundos"},{"name":"max_delay","desc":"delay maximo permitido en segundos"},{"name":"jitter","desc":"si True anade variacion aleatoria al delay"}],"output":"numero flotante representando el delay en segundos"}


id: convert_github_to_raw_url_py_core
name: convert_github_to_raw_url
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: convert_github_to_raw_url(url: str) -> str
tags: ["github","gitlab","url","raw","blob","convert","transform"]
uses_functions: []
uses_types: []
returns: []
imports: ["urllib.parse"]
test_file_path: python/functions/core/convert_github_to_raw_url_test.py
file_path: python/functions/core/convert_github_to_raw_url.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 17ad3b082e6b7ebce7539c97922d055b7c0206d6b6c2d674889059f1d58cb961
description:
  Convierte una URL de blob de GitHub/GitLab a su URL raw. Ej: github.com/org/repo/blob/main/file.py → raw.githubusercontent.com/org/repo/main/file.py. Retorna la URL sin cambios si no aplica.

example:
  ```python
  from core.convert_github_to_raw_url import convert_github_to_raw_url

  # GitHub blob → raw.githubusercontent.com
  url = convert_github_to_raw_url(
      "https://github.com/openai/whisper/blob/main/README.md"
  )
  # "https://raw.githubusercontent.com/openai/whisper/main/README.md"

  # GitLab blob → raw
  url = convert_github_to_raw_url(
      "https://gitlab.com/org/repo/-/blob/main/file.py"
  )
  # "https://gitlab.com/org/repo/-/raw/main/file.py"

  # URL sin blob → sin cambios
  url = convert_github_to_raw_url("https://github.com/org/repo")
  # "https://github.com/org/repo"
  ```

tests:
  ["URL GitHub blob","URL GitLab blob","URL que no es blob retorna sin cambios","URL no-GitHub retorna sin cambios"]

notes:
  Algoritmo:
  1. Parsear la URL con `urllib.parse.urlparse`.
  2. Si host es `github.com`: buscar segmento `blob` en el path.
     - Si existe: eliminar el segmento `blob` y cambiar el dominio a `raw.githubusercontent.com`.
  3. Si host es `gitlab.com` o empieza con `gitlab.`: reemplazar `/-/blob/` por `/-/raw/`
     o `/blob/` por `/raw/`.
  4. Cualquier otro host: retornar la URL sin cambios.

  Funcion pura. No hace I/O ni tiene efectos secundarios.

code:
  """Convierte URLs de blob de GitHub/GitLab a su equivalente raw."""

  from urllib.parse import urlparse, urlunparse


  def convert_github_to_raw_url(url: str) -> str:
      """Convierte una URL de blob de GitHub o GitLab a su URL raw.

      GitHub blob:
          https://github.com/org/repo/blob/main/path/file.py
          → https://raw.githubusercontent.com/org/repo/main/path/file.py

      GitLab blob:
          https://gitlab.com/org/repo/-/blob/main/path/file.py
          → https://gitlab.com/org/repo/-/raw/main/path/file.py

      Si la URL no contiene un path tipo blob, la retorna sin cambios.

      Args:
          url: URL de GitHub o GitLab, posiblemente apuntando a un blob.

      Returns:
          URL raw si aplica la transformacion; la URL original en caso contrario.
      """
      url = url.strip()
      if not url:
          return url

      parsed = urlparse(url)
      host = parsed.hostname or ""

      # --- GitHub ---
      if host in ("github.com", "www.github.com"):
          # Path tipico: /org/repo/blob/ref/path/to/file
          segments = parsed.path.split("/")
          if "blob" in segments:
              blob_idx = segments.index("blob")
              # Eliminar segmento "blob": /org/repo/ref/path/...
              new_segments = segments[:blob_idx] + segments[blob_idx + 1:]
              new_path = "/".join(new_segments)
              raw_url = urlunparse((
                  "https",
                  "raw.githubusercontent.com",
                  new_path,
                  parsed.params,
                  parsed.query,
                  parsed.fragment,
              ))
              return raw_url
          return url

      # --- GitLab ---
      if host in ("gitlab.com", "www.gitlab.com") or host.startswith("gitlab."):
          # Path tipico: /org/repo/-/blob/ref/path o /org/repo/blob/ref/path
          new_path = parsed.path.replace("/-/blob/", "/-/raw/").replace("/blob/", "/raw/")
          if new_path != parsed.path:
              raw_url = urlunparse((
                  parsed.scheme,
                  parsed.netloc,
                  new_path,
                  parsed.params,
                  parsed.query,
                  parsed.fragment,
              ))
              return raw_url
          return url

      # No aplica transformacion
      return url

params_schema:
  {"params":[{"name":"url","desc":"URL a convertir (puede ser GitHub, GitLab o cualquier otra)"}],"output":"string con la URL convertida a raw format (o sin cambios si no es un blob)"}


id: create_node_mapping_py_core
name: create_node_mapping
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def create_node_mapping(tree: list[dict]) -> dict[str, dict]
description: Crea dict plano node_id->node para lookup O(1) en un arbol jerarquico.
tags: ["tree","mapping","index","lookup"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Los valores son referencias a los nodos originales, no copias.
content_hash: e5fef7bd40be8bd1510317545639f46113e24e78f247d4a859c3937a2756e037
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  tree = [{"node_id": "0001", "title": "A", "nodes": [{"node_id": "0002", "title": "B"}]}]
  mapping = create_node_mapping(tree)
  mapping["0002"]["title"]  # "B"
  ```

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"tree","desc":"lista de dicts representando arbol jerarquico con campos node_id y nodes"}],"output":"dict con node_id como keys y nodos como values para lookup O(1)"}


id: cursor_paginate_py_core
name: cursor_paginate
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["pagination","cursor","retry","generic","api","backoff"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["time","typing.Callable","typing.TypeVar"]
test_file_path: python/functions/core/cursor_paginate_test.py
file_path: python/functions/core/cursor_paginate.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 47c1ed65683f24308ad060f6ff15cf2952e8d46cbdb51543a0809b9cca0dd6ec
signature:
  def cursor_paginate(fetch_page: Callable[..., list[T]], get_cursor: Callable[[T], str | None], page_size: int = 100, max_items: int = 2000, max_retries: int = 3, retry_delay: float = 2.0, retryable_exceptions: tuple[type[Exception], ...] = (ConnectionError, TimeoutError, OSError)) -> list[T]

description:
  Paginador generico basado en cursor que funciona con cualquier API que use cursor-based pagination. Cada pagina se obtiene con retry automatico con exponential backoff. Se detiene cuando la pagina esta vacia, el batch es menor que page_size, se alcanza max_items, o el cursor del ultimo item es None.

example:
  ```python
  from cursor_paginate import cursor_paginate

  def fetch_users(limit: int, cursor: str | None) -> list[dict]:
      params = {"limit": limit}
      if cursor:
          params["cursor"] = cursor
      return requests.get("https://api.example.com/users", params=params).json()["items"]

  def get_cursor(user: dict) -> str | None:
      return user.get("next_cursor")

  users = cursor_paginate(
      fetch_page=fetch_users,
      get_cursor=get_cursor,
      page_size=100,
      max_items=5000,
      max_retries=3,
      retry_delay=2.0,
  )
  ```

tests:
  ["API que retorna 3 paginas de 10 items","API que falla 1 vez por pagina (retry funciona)","max_items limita correctamente","API que retorna pagina parcial (ultima pagina)","Cursor None en ultimo item (se detiene)"]

notes:
  El caller solo necesita proveer dos callables:
  - `fetch_page(limit, cursor)`: recibe `limit` y `cursor` como kwargs, retorna lista de items.
  - `get_cursor(item)`: extrae el cursor del ultimo item de la pagina; retornar None indica fin de datos.

  El exponential backoff interno aplica `retry_delay * 2^attempt` sin jitter. Solo se reintentan las excepciones en `retryable_exceptions`; cualquier otra excepcion propaga inmediatamente.

  Condiciones de parada (cualquiera de ellas):
  1. La pagina retornada esta vacia.
  2. La pagina retornada tiene menos items que `page_size` (pagina parcial = ultima pagina).
  3. El total acumulado alcanza o supera `max_items` (se trunca y se para).
  4. `get_cursor(batch[-1])` retorna `None`.

  Funcion impura: llama a `fetch_page` que tipicamente hace I/O de red y usa `time.sleep` en los reintentos.

code:
  """Generic cursor-based paginator for any API that uses cursor pagination."""

  import time
  from typing import Callable, TypeVar

  T = TypeVar("T")


  def cursor_paginate(
      fetch_page: Callable[..., list[T]],
      get_cursor: Callable[[T], str | None],
      page_size: int = 100,
      max_items: int = 2000,
      max_retries: int = 3,
      retry_delay: float = 2.0,
      retryable_exceptions: tuple[type[Exception], ...] = (
          ConnectionError,
          TimeoutError,
          OSError,
      ),
  ) -> list[T]:
      """Paginate through a cursor-based API, collecting all items.

      Fetches pages one at a time by calling fetch_page with limit and cursor
      kwargs. Retries each page on transient errors using exponential backoff.
      Stops when a page is empty, a partial page is returned, max_items is
      reached, or the cursor from the last item is None.

      Args:
          fetch_page: Callable that accepts ``limit`` and ``cursor`` as keyword
              arguments and returns a list of items for that page.
          get_cursor: Callable that receives the last item of a page and returns
              the cursor string to use for the next page, or None if there are
              no more pages.
          page_size: Number of items to request per page.
          max_items: Hard cap on total items collected. Collection stops and the
              list is truncated once this limit is reached.
          max_retries: Maximum number of retry attempts per page after the first
              failure.
          retry_delay: Base delay in seconds between retries (doubled each
              attempt — exponential backoff without jitter).
          retryable_exceptions: Tuple of exception types that trigger a retry.
              Any other exception propagates immediately.

      Returns:
          List of all collected items, in the order they were returned by the
          API, truncated to max_items.

      Raises:
          Exception: Re-raises the last exception if all retries for a page are
              exhausted.
      """
      all_items: list[T] = []
      cursor: str | None = None

      while True:
          batch = _fetch_with_retry(
              fetch_page=fetch_page,
              page_size=page_size,
              cursor=cursor,
              max_retries=max_retries,
              retry_delay=retry_delay,
              retryable_exceptions=retryable_exceptions,
          )

          if not batch:
              break

          all_items.extend(batch)

          if len(all_items) >= max_items:
              del all_items[max_items:]
              break

          if len(batch) < page_size:
              break

          cursor = get_cursor(batch[-1])
          if cursor is None:
              break

      return all_items


  def _fetch_with_retry(
      fetch_page: Callable[..., list[T]],
      page_size: int,
      cursor: str | None,
      max_retries: int,
      retry_delay: float,
      retryable_exceptions: tuple[type[Exception], ...],
  ) -> list[T]:
      """Call fetch_page once, retrying on retryable_exceptions with exponential backoff."""
      last_exc: Exception | None = None
      for attempt in range(max_retries + 1):
          try:
              return fetch_page(limit=page_size, cursor=cursor)
          except retryable_exceptions as exc:
              last_exc = exc
              if attempt >= max_retries:
                  raise
              delay = retry_delay * (2 ** attempt)
              time.sleep(delay)

      raise last_exc  # unreachable; satisfies type checkers

params_schema:
  {"params":[{"name":"fetch_page","desc":"callable que recibe limit y cursor como kwargs, retorna lista de items"},{"name":"get_cursor","desc":"callable que recibe un item y retorna su cursor (o None si es el ultimo)"},{"name":"page_size","desc":"numero de items por pagina a solicitar"},{"name":"max_items","desc":"limite maximo total de items a acumular"},{"name":"max_retries","desc":"numero maximo de reintentos por pagina"},{"name":"retry_delay","desc":"delay inicial en segundos para exponential backoff entre reintentos"},{"name":"retryable_exceptions","desc":"tupla de tipos de excepcion que desencadenan reintento"}],"output":"lista de items recopilados de todas las paginas"}


id: detect_headings_by_font_py_core
name: detect_headings_by_font
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def detect_headings_by_font(pdf, min_delta: float = 2.0, max_levels: int = 4) -> list[dict]
tags: ["pdf","headings","font","detection","parsing","pdfplumber"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["pdfplumber","collections"]
tests: []
file_path: python/functions/core/detect_headings_by_font.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 1c28fd0465c425216674ea26a9bd3589423dac872b85b05c9346fea956f05db7
description:
  Detecta headings en un PDF analizando la distribucion de font sizes. El font size mas comun es el body; sizes significativamente mayores se clasifican como heading levels. Filtra headers/footers repetitivos.

example:
  ```python
  import pdfplumber
  from detect_headings_by_font import detect_headings_by_font

  with pdfplumber.open("document.pdf") as pdf:
      headings = detect_headings_by_font(pdf, min_delta=2.0, max_levels=4)
      for h in headings:
          print(f"Page {h['page_num']}: {'#' * h['level']} {h['title']}")
  ```

notes:
  Samplea cada 5ta pagina para construir el Counter de font sizes (optimizacion de rendimiento). El body_size es el font size mas frecuente. Los heading sizes deben ser >= body_size + min_delta Y tener frecuencia < 50% del body. Se limita a max_levels heading sizes ordenados desc (el mas grande = nivel 1). Titulos que aparecen en >30% de paginas son considerados headers/footers y se eliminan. Impure porque accede al estado interno de un objeto PDF ya abierto.

code:
  """Detect headings in a PDF by analyzing font size distribution."""

  from collections import Counter

  import pdfplumber


  def detect_headings_by_font(
      pdf: pdfplumber.PDF,
      min_delta: float = 2.0,
      max_levels: int = 4,
  ) -> list[dict]:
      """Detect headings by analyzing font size distribution across pages.

      The most common font size is treated as body text. Font sizes significantly
      larger than body (by at least min_delta) and appearing in fewer than 50% of
      chars are classified as heading levels.

      Args:
          pdf: An open pdfplumber.PDF object.
          min_delta: Minimum size difference above body size to qualify as heading.
          max_levels: Maximum number of heading levels to detect.

      Returns:
          list[dict]: List of {"level": int, "title": str, "page_num": int}
                      sorted by page number. Returns empty list if no headings detected.
      """
      if not pdf.pages:
          return []

      # Step 1: Sample font sizes from every 5th page to determine body size
      size_counter: Counter = Counter()
      sample_pages = [pdf.pages[i] for i in range(0, len(pdf.pages), 5)]
      if not sample_pages:
          sample_pages = [pdf.pages[0]]

      for page in sample_pages:
          try:
              chars = page.chars
              for ch in chars:
                  size = ch.get("size")
                  if size is not None:
                      size_counter[round(float(size), 1)] += 1
          except Exception:
              continue

      if not size_counter:
          return []

      # Step 2: Determine body size (most common font size)
      body_size, body_count = size_counter.most_common(1)[0]

      # Step 3: Identify heading sizes
      # Must be >= body_size + min_delta and frequency < 50% of body count
      heading_sizes = sorted(
          [
              size
              for size, count in size_counter.items()
              if size >= body_size + min_delta and count < body_count * 0.5
          ],
          reverse=True,
      )[:max_levels]

      if not heading_sizes:
          return []

      # Build size -> level mapping
      size_to_level = {size: i + 1 for i, size in enumerate(heading_sizes)}

      # Step 4: Collect heading text per page
      raw_headings: list[dict] = []
      total_pages = len(pdf.pages)

      for page_idx, page in enumerate(pdf.pages):
          page_num = page_idx + 1
          try:
              chars = page.chars
          except Exception:
              continue

          # Group consecutive chars of same heading size into text blocks
          current_size = None
          current_text = []

          for ch in chars:
              size = ch.get("size")
              if size is None:
                  continue
              rounded = round(float(size), 1)
              if rounded in size_to_level:
                  if rounded == current_size:
                      current_text.append(ch.get("text", ""))
                  else:
                      if current_text and current_size is not None:
                          text = "".join(current_text).strip()
                          if text:
                              raw_headings.append({
                                  "level": size_to_level[current_size],
                                  "title": text,
                                  "page_num": page_num,
                              })
                      current_size = rounded
                      current_text = [ch.get("text", "")]
              else:
                  if current_text and current_size is not None:
                      text = "".join(current_text).strip()
                      if text:
                          raw_headings.append({
                              "level": size_to_level[current_size],
                              "title": text,
                              "page_num": page_num,
                          })
                  current_size = None
                  current_text = []

          # Flush remaining
          if current_text and current_size is not None:
              text = "".join(current_text).strip()
              if text:
                  raw_headings.append({
                      "level": size_to_level[current_size],
                      "title": text,
                      "page_num": page_num,
                  })

      if not raw_headings:
          return []

      # Step 5: Deduplicate — remove titles appearing on > 30% of pages (headers/footers)
      title_page_counts: Counter = Counter(h["title"] for h in raw_headings)
      threshold = total_pages * 0.3

      filtered = [h for h in raw_headings if title_page_counts[h["title"]] <= threshold]

      return filtered

params_schema:
  {"params":[{"name":"pdf","desc":"objeto pdfplumber.PDF abierto con which_pages="},{"name":"min_delta","desc":"diferencia minima de font size para clasificar como heading"},{"name":"max_levels","desc":"numero maximo de heading levels a detectar"}],"output":"lista de dicts con campos page_num, level, title para cada heading detectado"}


id: detect_url_type_py_core
name: detect_url_type
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: detect_url_type(url: str, timeout: float = 10.0) -> tuple[str, dict]
tags: ["url","content-type","http","detect","classification","head-request"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["urllib.parse","httpx"]
test_file_path: python/functions/core/detect_url_type_test.py
file_path: python/functions/core/detect_url_type.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 87773e42c608fe028d35d537dba56f9afe6b6fdcbf2892c4eee30397154b5bed
description:
  Detecta el tipo de contenido de una URL. Retorna tipo ('webpage', 'pdf', 'markdown', 'text', 'code_repository') y metadata. Hace HTTP HEAD request solo si no puede determinarse por patron o extension.

example:
  ```python
  from core.detect_url_type import detect_url_type

  # Por patron URL (sin HTTP request)
  url_type, meta = detect_url_type("https://github.com/openai/whisper")
  # url_type = "code_repository", meta = {"detection": "url_pattern", ...}

  # Por extension (sin HTTP request)
  url_type, meta = detect_url_type("https://example.com/doc.pdf")
  # url_type = "pdf", meta = {"detection": "extension", ...}

  # Por HTTP HEAD request (cuando no se puede determinar sin red)
  url_type, meta = detect_url_type("https://example.com/page")
  # url_type = "webpage", meta = {"detection": "content_type_header", "content_type": "text/html", ...}
  ```

tests:
  ["URL .pdf por extension","URL github repo","URL markdown por extension","URL SSH git","URL .html por extension"]

notes:
  Algoritmo en orden de prioridad:
  1. SSH git shorthand (`git@host:path`) → `code_repository` inmediatamente.
  2. Patron URL de repos conocidos (github.com/org/repo, gitlab.com/org/repo) → `code_repository`.
  3. Extension del path de la URL (.pdf, .md, .txt, .html, .git) → tipo correspondiente.
  4. HTTP HEAD request → leer `Content-Type` header.
  5. Default: `"webpage"`.

  Hosts reconocidos como repos de codigo: github.com, gitlab.com, bitbucket.org, codeberg.org.

  Sub-recursos (issues, pulls, blob, tree, etc.) NO se clasifican como `code_repository`.

  Lanza `Exception` con mensaje descriptivo si el HEAD request falla (timeout, DNS, red).

code:
  """Detecta el tipo de contenido de una URL (webpage, pdf, markdown, text, code_repository)."""

  import re
  from urllib.parse import urlparse


  # Patrones de repos de codigo por hostname
  _CODE_REPO_HOSTS = {"github.com", "gitlab.com", "bitbucket.org", "codeberg.org"}

  # Extensiones reconocidas → tipo
  _EXT_TYPE_MAP = {
      ".pdf": "pdf",
      ".md": "markdown",
      ".markdown": "markdown",
      ".rst": "text",
      ".txt": "text",
      ".html": "webpage",
      ".htm": "webpage",
      ".xml": "text",
      ".json": "text",
      ".csv": "text",
      ".py": "text",
      ".js": "text",
      ".ts": "text",
      ".go": "text",
      ".rs": "text",
      ".cpp": "text",
      ".c": "text",
      ".java": "text",
      ".rb": "text",
      ".git": "code_repository",
  }

  # Content-Type header prefixes → tipo
  _CONTENT_TYPE_MAP = {
      "application/pdf": "pdf",
      "text/markdown": "markdown",
      "text/x-markdown": "markdown",
      "text/plain": "text",
      "text/html": "webpage",
      "text/xml": "text",
      "application/xml": "text",
      "application/json": "text",
  }


  def _is_code_repo_url(parsed, path_segments: list[str]) -> bool:
      """Return True si la URL apunta a la raiz de un repositorio de codigo."""
      host = parsed.hostname or ""
      if host not in _CODE_REPO_HOSTS:
          return False
      # Acepta org/repo o org/repo/ o org/repo.git (2 segmentos minimos)
      if len(path_segments) < 2:
          return False
      # Rechaza sub-recursos conocidos: issues, pulls, blob, tree, releases, etc.
      _SUB_RESOURCES = {"issues", "pulls", "blob", "tree", "releases", "tags",
                        "commits", "compare", "wiki", "discussions", "actions",
                        "security", "pulse", "graphs", "-", "settings"}
      if len(path_segments) >= 3 and path_segments[2].rstrip(".git") in _SUB_RESOURCES:
          return False
      return True


  def _is_ssh_git_url(url: str) -> bool:
      """Return True si la URL es un SSH git shorthand (git@host:path)."""
      return url.strip().startswith("git@")


  def _type_from_extension(path: str) -> str | None:
      """Detecta tipo segun la extension del path de la URL. Retorna None si no aplica."""
      # Ignorar query string / fragment
      clean_path = path.split("?")[0].split("#")[0]
      for ext, url_type in _EXT_TYPE_MAP.items():
          if clean_path.lower().endswith(ext):
              return url_type
      return None


  def _type_from_content_type(content_type_header: str) -> str:
      """Mapea un Content-Type header al tipo de URL."""
      ct = content_type_header.lower().split(";")[0].strip()
      for prefix, url_type in _CONTENT_TYPE_MAP.items():
          if ct.startswith(prefix):
              return url_type
      return "webpage"


  def detect_url_type(url: str, timeout: float = 10.0) -> tuple[str, dict]:
      """Detecta el tipo de contenido de una URL.

      Algoritmo:
      1. Verificar si la URL es un patron de repo de codigo (git@, github.com/org/repo).
      2. Verificar extension en el path de la URL (.pdf, .md, .txt, .html, .git).
      3. Si no se determino: HTTP HEAD request para leer Content-Type header.
      4. Default: "webpage".

      Args:
          url: URL a analizar.
          timeout: Timeout en segundos para el HTTP HEAD request (si es necesario).

      Returns:
          Tuple de (tipo, metadata) donde tipo es uno de:
          "webpage", "pdf", "markdown", "text", "code_repository".
          metadata incluye la informacion disponible (extension, content_type, etc.).

      Raises:
          Exception: Si falla la conexion HTTP cuando es necesaria.
      """
      import httpx

      url = url.strip()
      metadata: dict = {"url": url}

      # 1. SSH git shorthand
      if _is_ssh_git_url(url):
          metadata["detection"] = "ssh_pattern"
          return "code_repository", metadata

      parsed = urlparse(url)
      path_segments = [s for s in parsed.path.split("/") if s]

      # 2. Code repo by URL pattern
      if _is_code_repo_url(parsed, path_segments):
          metadata["detection"] = "url_pattern"
          metadata["host"] = parsed.hostname
          return "code_repository", metadata

      # 3. Extension-based detection
      ext_type = _type_from_extension(parsed.path)
      if ext_type is not None:
          metadata["detection"] = "extension"
          metadata["path"] = parsed.path
          return ext_type, metadata

      # 4. HTTP HEAD request
      try:
          response = httpx.head(url, timeout=timeout, follow_redirects=True)
          content_type = response.headers.get("content-type", "")
          metadata["detection"] = "content_type_header"
          metadata["content_type"] = content_type
          metadata["status_code"] = response.status_code
          return _type_from_content_type(content_type), metadata
      except Exception as exc:
          raise Exception(f"detect_url_type: HEAD request failed for {url!r}: {exc}") from exc

params_schema:
  {"params":[{"name":"url","desc":"URL a clasificar"},{"name":"timeout","desc":"timeout en segundos para HTTP HEAD request si es necesario"}],"output":"tupla (tipo, metadata_dict) donde tipo es string como 'pdf', 'webpage', 'code_repository' etc."}


id: docx_to_markdown_py_core
name: docx_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: docx_to_markdown(docx_path: str) -> str
tags: ["docx","markdown","word","conversion","document","parsing","text"]
uses_functions: ["format_table_to_markdown_py_core"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["python-docx","lxml"]
test_file_path: python/functions/core/docx_to_markdown_test.py
file_path: python/functions/core/docx_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 968aa7b9bb823432f6a42f25c92de160ecc51cedb524cd50ab13a01f221e0e13
description:
  Convierte un documento Word (.docx) a markdown preservando estructura (headings), formato inline (bold, italic, underline) y tablas en su posicion original.

example:
  ```python
  md = docx_to_markdown("informe.docx")
  # # Titulo
  #
  # Primer parrafo.
  #
  # | Col1 | Col2 |
  # | ---- | ---- |
  # | a    | b    |
  #
  # Parrafo despues de la tabla.
  ```

tests:
  ["docx con headings y parrafos","docx con tablas intercaladas","docx con formato bold/italic","docx vacio","archivo no encontrado lanza FileNotFoundError"]

notes:
  Recorre `doc.element.body` en orden (no `doc.paragraphs` + `doc.tables` por separado) para preservar la posicion original de las tablas. Construye un mapa `{id(tbl_element): Table}` para lookup O(1). El formato inline aplica underline (`<ins>`), italic (`*`) y bold (`**`) en ese orden de mas interno a mas externo. Los headings se detectan por el estilo del parrafo (`Heading 1`, `Heading 2`, etc.). Requiere `python-docx` instalado en el entorno.

code:
  """Convert a Word .docx document to Markdown, preserving structure, inline
  formatting and tables in their original document order."""

  import os
  from lxml import etree

  from format_table_to_markdown import format_table_to_markdown


  # XML namespace used by python-docx element tags
  _W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
  _TAG_P = f"{{{_W}}}p"
  _TAG_TBL = f"{{{_W}}}tbl"
  _TAG_TR = f"{{{_W}}}tr"
  _TAG_TC = f"{{{_W}}}tc"
  _TAG_R = f"{{{_W}}}r"
  _TAG_T = f"{{{_W}}}t"
  _TAG_RPR = f"{{{_W}}}rPr"
  _TAG_B = f"{{{_W}}}b"
  _TAG_I = f"{{{_W}}}i"
  _TAG_U = f"{{{_W}}}u"
  _TAG_PSTYLE = f"{{{_W}}}pStyle"
  _TAG_PPR = f"{{{_W}}}pPr"


  def _heading_level(paragraph) -> int:
      """Return heading level (1-6) if the paragraph is a heading, else 0."""
      pPr = paragraph._p.find(_TAG_PPR)
      if pPr is None:
          return 0
      pStyle = pPr.find(_TAG_PSTYLE)
      if pStyle is None:
          return 0
      val = pStyle.get(f"{{{_W}}}val", "")
      if val.lower().startswith("heading"):
          parts = val.split()
          if len(parts) == 2:
              try:
                  return int(parts[1])
              except ValueError:
                  pass
          # Some locales use "Heading1" (no space)
          suffix = val[len("heading"):]
          if suffix.isdigit():
              return int(suffix)
      return 0


  def _run_to_md(run_elem) -> str:
      """Convert a single <w:r> element to a markdown-formatted string."""
      # Collect text
      text_parts = []
      for t in run_elem.findall(_TAG_T):
          text_parts.append(t.text or "")
      text = "".join(text_parts)
      if not text:
          return ""

      # Read formatting from <w:rPr>
      rPr = run_elem.find(_TAG_RPR)
      bold = False
      italic = False
      underline = False
      if rPr is not None:
          bold = rPr.find(_TAG_B) is not None
          italic = rPr.find(_TAG_I) is not None
          u_elem = rPr.find(_TAG_U)
          if u_elem is not None:
              u_val = u_elem.get(f"{{{_W}}}val", "")
              underline = u_val not in ("none", "")

      # Apply markdown formatting (innermost first: underline → italic → bold)
      if underline:
          text = f"<ins>{text}</ins>"
      if italic:
          text = f"*{text}*"
      if bold:
          text = f"**{text}**"
      return text


  def _paragraph_to_md(paragraph) -> str:
      """Convert a python-docx Paragraph to a markdown string."""
      level = _heading_level(paragraph)
      runs_md = "".join(
          _run_to_md(elem)
          for elem in paragraph._p
          if elem.tag == _TAG_R
      )
      if level:
          return f"{'#' * level} {runs_md}"
      return runs_md


  def _table_to_md(table) -> str:
      """Convert a python-docx Table to a markdown table string."""
      rows: list[list[str]] = []
      for row in table.rows:
          cells = []
          for cell in row.cells:
              # Join all paragraphs in the cell with a space
              cell_text = " ".join(p.text for p in cell.paragraphs).strip()
              cells.append(cell_text)
          rows.append(cells)
      return format_table_to_markdown(rows, has_header=True)


  def docx_to_markdown(docx_path: str) -> str:
      """Convert a Word .docx document to Markdown.

      Preserves document structure (headings), inline formatting (bold, italic,
      underline) and tables in their original position.

      Args:
          docx_path: Absolute or relative path to the .docx file.

      Returns:
          Markdown string representing the document.

      Raises:
          FileNotFoundError: If the file does not exist.
          Exception: If the file cannot be parsed as a .docx document.
      """
      import docx  # deferred so the module is importable without python-docx installed

      if not os.path.exists(docx_path):
          raise FileNotFoundError(f"File not found: {docx_path}")

      doc = docx.Document(docx_path)

      # Build a mapping from the XML element id to the Table object for O(1) lookup
      table_map: dict[int, object] = {
          id(table._tbl): table for table in doc.tables
      }

      parts: list[str] = []

      for child in doc.element.body:
          if child.tag == _TAG_P:
              # Wrap in a temporary paragraph object to reuse _paragraph_to_md
              from docx.text.paragraph import Paragraph
              para = Paragraph(child, doc)
              md = _paragraph_to_md(para)
              if md.strip():
                  parts.append(md)
          elif child.tag == _TAG_TBL:
              table = table_map.get(id(child))
              if table is not None:
                  md = _table_to_md(table)
                  if md:
                      parts.append(md)

      return "\n\n".join(parts)

params_schema:
  {"params":[{"name":"docx_path","desc":"ruta al archivo Word .docx"}],"output":"string markdown con headings, parrafos formateados y tablas preservadas"}


id: drop_py_core
name: drop
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def drop(xs: list, n: int) -> list
description: Descarta los primeros n elementos de una lista.
tags: ["list","functional","drop","slice","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = drop([1, 2, 3, 4, 5], 2)
  # [3, 4, 5]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Si n > len(xs), retorna lista vacia. No muta la original.
content_hash: 59cdd16c1737ede669c2f776a14eccf9f880c6d95eb5e28101b0e8b51848dd42
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista de la que descartar elementos"},{"name":"n","desc":"numero de elementos a descartar desde el inicio"}],"output":"lista sin los primeros n elementos"}


id: epub_to_markdown_py_core
name: epub_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def epub_to_markdown(epub_path: str) -> str
tags: ["epub","markdown","ebook","parsing","conversion","html","text-extraction"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["zipfile","html","re","ebooklib"]
test_file_path: python/functions/core/epub_to_markdown_test.py
file_path: python/functions/core/epub_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: f0083d088423b21fa96bb31e06f5513b554c04ce543a301b910c6317c96fd9e4
description:
  Convierte un ebook EPUB a markdown. Intenta ebooklib primero para extraccion estructurada (titulo, autor, documentos); fallback a extraccion manual con zipfile si ebooklib no esta instalado.

example:
  ```python
  md = epub_to_markdown("/path/to/book.epub")
  print(md[:500])
  # # Mi Libro
  # **Author:** Ana Perez
  #
  # # Introduccion
  # Primer parrafo...
  ```

tests:
  ["conversion de headings h1-h3","conversion de bold e italic","script y style se eliminan del output","HTML entities se convierten a caracteres","epub sin ebooklib extrae texto de archivos html","epub con ebooklib incluye titulo y autor en el output","epub corrupto lanza excepcion"]

notes:
  Conversion HTML a markdown cubre: headings h1-h6, bold (`<strong>`/`<b>`), italic (`<em>`/`<i>`), paragraphs, line breaks. Elimina `<script>` y `<style>`. Desescapa entidades HTML y normaliza whitespace.

  Con ebooklib: extrae metadata DC (titulo, autor) del OPF y procesa solo los ITEM_DOCUMENT del spine.

  Sin ebooklib (fallback ZIP): lista archivos `.html`/`.xhtml`/`.htm` en orden alfabetico y extrae su contenido. No hay metadata de titulo/autor en este modo.

  Dependencia opcional: `pip install ebooklib`. Si no esta instalada la funcion sigue funcionando via zipfile.

  Reimplementacion conceptual desde OpenViking `openviking/parse/parsers/epub.py` (AGPL-3.0). El codigo es original.

code:
  """Convert an EPUB file to markdown text."""

  import re
  import zipfile
  from html import unescape
  from html.parser import HTMLParser


  def _remove_tags(html: str, tag: str) -> str:
      """Remove a tag and its content from HTML string."""
      pattern = re.compile(rf'<{tag}[^>]*>.*?</{tag}>', re.IGNORECASE | re.DOTALL)
      return pattern.sub('', html)


  def _html_to_markdown(html: str) -> str:
      """Convert basic HTML to markdown.

      Handles headings, bold, italic, paragraphs, line breaks
      and strips remaining tags.

      Args:
          html: HTML string to convert.

      Returns:
          Markdown-formatted string.
      """
      # Remove script and style blocks
      text = _remove_tags(html, 'script')
      text = _remove_tags(text, 'style')

      # Headings h1-h6
      for level in range(6, 0, -1):
          hashes = '#' * level
          text = re.sub(
              rf'<h{level}[^>]*>(.*?)</h{level}>',
              lambda m, h=hashes: f'{h} {m.group(1).strip()}',
              text,
              flags=re.IGNORECASE | re.DOTALL,
          )

      # Bold
      text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.IGNORECASE | re.DOTALL)
      text = re.sub(r'<b[^>]*>(.*?)</b>', r'**\1**', text, flags=re.IGNORECASE | re.DOTALL)

      # Italic
      text = re.sub(r'<em[^>]*>(.*?)</em>', r'*\1*', text, flags=re.IGNORECASE | re.DOTALL)
      text = re.sub(r'<i[^>]*>(.*?)</i>', r'*\1*', text, flags=re.IGNORECASE | re.DOTALL)

      # Paragraphs — append double newline after content
      text = re.sub(r'<p[^>]*>(.*?)</p>', lambda m: m.group(1).strip() + '\n\n', text, flags=re.IGNORECASE | re.DOTALL)

      # Line breaks
      text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)

      # Strip remaining HTML tags
      text = re.sub(r'<[^>]+>', '', text)

      # Unescape HTML entities
      text = unescape(text)

      # Normalize whitespace: collapse multiple blank lines into two
      text = re.sub(r'\n{3,}', '\n\n', text)
      text = re.sub(r'[ \t]+', ' ', text)

      return text.strip()


  def _epub_via_ebooklib(epub_path: str) -> str:
      """Extract markdown from EPUB using ebooklib."""
      import ebooklib
      from ebooklib import epub

      book = epub.read_epub(epub_path)

      # Metadata
      title_meta = book.get_metadata('DC', 'title')
      author_meta = book.get_metadata('DC', 'creator')
      title = title_meta[0][0] if title_meta else 'Unknown Title'
      author = author_meta[0][0] if author_meta else 'Unknown Author'

      parts = [f'# {title}', f'**Author:** {author}']

      for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
          content = item.get_content().decode('utf-8', errors='replace')
          md = _html_to_markdown(content)
          if md:
              parts.append(md)

      return '\n\n'.join(parts)


  def _epub_via_zipfile(epub_path: str) -> str:
      """Extract markdown from EPUB using zipfile (fallback)."""
      parts = []
      with zipfile.ZipFile(epub_path, 'r') as zf:
          html_files = sorted(
              name for name in zf.namelist()
              if name.lower().endswith(('.html', '.xhtml', '.htm'))
          )
          for name in html_files:
              raw = zf.read(name).decode('utf-8', errors='replace')
              md = _html_to_markdown(raw)
              if md:
                  parts.append(md)

      return '\n\n'.join(parts)


  def epub_to_markdown(epub_path: str) -> str:
      """Convert an EPUB ebook to markdown.

      Attempts to use ebooklib for structured extraction (title, author,
      document items). Falls back to manual ZIP extraction if ebooklib is
      not installed.

      Args:
          epub_path: Path to the .epub file.

      Returns:
          Markdown string with the book content.

      Raises:
          Exception: If the file cannot be read or is not a valid EPUB.
      """
      try:
          return _epub_via_ebooklib(epub_path)
      except ImportError:
          return _epub_via_zipfile(epub_path)

params_schema:
  {"params":[{"name":"epub_path","desc":"ruta al archivo EPUB"}],"output":"string markdown con titulo, autor (si esta disponible) y contenido del ebook"}


id: estimate_token_count_py_core
name: estimate_token_count
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def estimate_token_count(content: str) -> int
tags: ["tokens","estimation","nlp","cjk","text"]
uses_functions: []
uses_types: []
returns: []
imports: ["re"]
tests: ["texto vacio retorna cero","solo latin","solo CJK","texto mixto"]
test_file_path: python/functions/core/parse_markdown_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: bad61df097deb73febc71ce39d69f4473925cdc6620e538cd0a1c9ccdfa4488b
description:
  Estimacion rapida de tokens sin tokenizer. CJK chars cuentan ~0.7 token/char, otros non-whitespace ~0.3 token/char.

example:
  ```python
  estimate_token_count("hello world")  # 3
  estimate_token_count("中文语")        # 2  (3 * 0.7 = 2)
  estimate_token_count("")             # 0
  ```

notes:
  Funcion pura. No requiere ninguna dependencia externa. Precision aproximada: util para guardianes de limite de contexto antes de llamar a LLMs, no para conteo exacto de tokens BPE. CJK range: `[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]` (CJK unificado, Hiragana/Katakana, Hangul).

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"content","desc":"string de contenido a contar"}],"output":"entero estimando el numero de tokens en el contenido"}


id: excel_to_markdown_py_core
name: excel_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: excel_to_markdown(path: str, max_rows_per_sheet: int = 1000) -> str
tags: ["excel","markdown","xlsx","xls","conversion","parser","io"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["openpyxl","xlrd"]
test_file_path: python/functions/core/excel_to_markdown_test.py
file_path: python/functions/core/excel_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 298367fec629a02550817c1ea82b7ce0dbe79d93cbe6ac75b792387e57af41a3
description:
  Convierte un archivo Excel (.xlsx, .xls, .xlsm) a markdown con cada sheet como seccion H2. Soporta tipos de celda: fechas ISO, booleanos, errores Excel, numeros enteros y flotantes. Trunca sheets que superen max_rows_per_sheet.

example:
  ```python
  from excel_to_markdown import excel_to_markdown

  md = excel_to_markdown("report.xlsx")
  print(md)
  # ## Sheet: Ventas
  #
  # **Dimensions:** 101 x 4
  #
  # | Producto | Precio | Cantidad | Total |
  # | --- | --- | --- | --- |
  # | Manzana | 1 | 100 | 100 |
  # ...

  # Con limite de filas
  md = excel_to_markdown("big_file.xlsx", max_rows_per_sheet=50)
  ```

tests:
  ["xlsx con multiples sheets produce una seccion H2 por sheet","sheet vacio produce nota de sheet vacio","sheet truncado con nota de filas omitidas","sheet con formulas data_only muestra valores calculados","extension no soportada lanza ValueError","archivo inexistente lanza FileNotFoundError","dimensiones del sheet en metadata","tabla markdown con formato correcto"]

notes:
  - `.xlsx` y `.xlsm`: usa `openpyxl` con `data_only=True` (lee valores calculados, no formulas).
  - `.xls` (legacy): usa `xlrd`. Manejo de tipos especiales: EMPTY/BLANK → "", DATE → ISO 8601, BOOLEAN → "TRUE"/"FALSE", ERROR → codigo Excel (#NULL!, #DIV/0!, etc.), NUMBER → entero si no tiene decimales.
  - Fechas sin hora se formatean como `YYYY-MM-DD`; con hora como `YYYY-MM-DDTHH:MM:SS`.
  - Los pipes `|` dentro de celdas se escapan como `\|`.
  - Si `xlwt` no esta disponible, los tests .xls se saltan (xlwt solo se necesita para crear fixtures, no para leer).
  - Reimplementacion desde cero, inspirada conceptualmente en OpenViking (AGPL-3.0). Sin codigo copiado.

code:
  """Convierte archivos Excel a Markdown con cada sheet como seccion H2."""

  import os
  from pathlib import Path


  # Codigos de error Excel para xlrd
  _XL_ERROR_CODES = {
      0: "#NULL!",
      7: "#DIV/0!",
      15: "#VALUE!",
      23: "#REF!",
      29: "#NAME?",
      36: "#NUM!",
      42: "#N/A",
  }


  def _rows_to_markdown_table(rows: list[list[str]]) -> str:
      """Convierte filas de strings a tabla markdown."""
      if not rows:
          return ""

      header = rows[0]
      col_count = len(header)

      # Normalizar todas las filas al mismo numero de columnas
      normalized = []
      for row in rows:
          if len(row) < col_count:
              row = row + [""] * (col_count - len(row))
          normalized.append(row[:col_count])

      # Escapar pipes en celdas
      def escape(cell: str) -> str:
          return cell.replace("|", "\\|").replace("\n", " ")

      lines = []
      # Header
      lines.append("| " + " | ".join(escape(c) for c in normalized[0]) + " |")
      # Separator
      lines.append("| " + " | ".join("---" for _ in range(col_count)) + " |")
      # Data rows
      for row in normalized[1:]:
          lines.append("| " + " | ".join(escape(c) for c in row) + " |")

      return "\n".join(lines)


  def _cell_value_xlrd(cell, workbook) -> str:
      """Convierte una celda xlrd a string segun su tipo."""
      import xlrd

      ctype = cell.ctype

      if ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
          return ""
      elif ctype == xlrd.XL_CELL_DATE:
          try:
              dt = xlrd.xldate_as_datetime(cell.value, workbook.datemode)
              if dt.hour == 0 and dt.minute == 0 and dt.second == 0:
                  return dt.date().isoformat()
              return dt.isoformat()
          except Exception:
              return str(cell.value)
      elif ctype == xlrd.XL_CELL_BOOLEAN:
          return "TRUE" if cell.value else "FALSE"
      elif ctype == xlrd.XL_CELL_ERROR:
          return _XL_ERROR_CODES.get(int(cell.value), "#ERROR!")
      elif ctype == xlrd.XL_CELL_NUMBER:
          v = cell.value
          if v == int(v):
              return str(int(v))
          return str(v)
      elif ctype == xlrd.XL_CELL_TEXT:
          return str(cell.value)
      else:
          return str(cell.value)


  def _sheet_xlrd(sheet, workbook, max_rows: int) -> str:
      """Convierte un sheet xlrd a markdown."""
      nrows = sheet.nrows
      ncols = sheet.ncols

      lines = []
      lines.append(f"## Sheet: {sheet.name}")
      lines.append("")
      lines.append(f"**Dimensions:** {nrows} x {ncols}")
      lines.append("")

      if nrows == 0 or ncols == 0:
          lines.append("*(empty sheet)*")
          return "\n".join(lines)

      display_rows = min(nrows, max_rows)
      rows = []
      for r in range(display_rows):
          row_data = [_cell_value_xlrd(sheet.cell(r, c), workbook) for c in range(ncols)]
          rows.append(row_data)

      lines.append(_rows_to_markdown_table(rows))

      if nrows > max_rows:
          omitted = nrows - max_rows
          lines.append("")
          lines.append(f"*{omitted} rows omitted (max_rows_per_sheet={max_rows})*")

      return "\n".join(lines)


  def _cell_value_openpyxl(cell) -> str:
      """Convierte una celda openpyxl a string."""
      v = cell.value
      if v is None:
          return ""
      if isinstance(v, bool):
          return "TRUE" if v else "FALSE"
      if isinstance(v, float):
          if v == int(v):
              return str(int(v))
          return str(v)
      if isinstance(v, int):
          return str(v)
      # Fechas y datetimes
      import datetime
      if isinstance(v, datetime.datetime):
          if v.hour == 0 and v.minute == 0 and v.second == 0:
              return v.date().isoformat()
          return v.isoformat()
      if isinstance(v, datetime.date):
          return v.isoformat()
      return str(v)


  def _sheet_openpyxl(ws, max_rows: int) -> str:
      """Convierte un worksheet openpyxl a markdown."""
      all_rows = list(ws.iter_rows())
      nrows = len(all_rows)
      ncols = ws.max_column or 0

      lines = []
      lines.append(f"## Sheet: {ws.title}")
      lines.append("")
      lines.append(f"**Dimensions:** {nrows} x {ncols}")
      lines.append("")

      if nrows == 0 or ncols == 0:
          lines.append("*(empty sheet)*")
          return "\n".join(lines)

      display_rows = min(nrows, max_rows)
      rows = []
      for row in all_rows[:display_rows]:
          row_data = [_cell_value_openpyxl(cell) for cell in row]
          rows.append(row_data)

      lines.append(_rows_to_markdown_table(rows))

      if nrows > max_rows:
          omitted = nrows - max_rows
          lines.append("")
          lines.append(f"*{omitted} rows omitted (max_rows_per_sheet={max_rows})*")

      return "\n".join(lines)


  def excel_to_markdown(path: str, max_rows_per_sheet: int = 1000) -> str:
      """Convierte un archivo Excel (.xlsx, .xls, .xlsm) a markdown.

      Cada sheet se convierte en una seccion H2. Las filas se representan
      como tablas markdown. Si el numero de filas supera max_rows_per_sheet,
      el sheet se trunca y se añade una nota.

      Args:
          path: Ruta al archivo Excel (.xlsx, .xls, .xlsm).
          max_rows_per_sheet: Maximo de filas a incluir por sheet (default 1000).

      Returns:
          String markdown con todos los sheets del archivo.

      Raises:
          ValueError: Si la extension no es soportada.
          FileNotFoundError: Si el archivo no existe.
          Exception: Si hay errores leyendo el archivo.
      """
      p = Path(path)
      if not p.exists():
          raise FileNotFoundError(f"File not found: {path}")

      ext = p.suffix.lower()

      if ext == ".xls":
          import xlrd
          wb = xlrd.open_workbook(path)
          sections = []
          for sheet_name in wb.sheet_names():
              sheet = wb.sheet_by_name(sheet_name)
              sections.append(_sheet_xlrd(sheet, wb, max_rows_per_sheet))
          return "\n\n".join(sections)

      elif ext in (".xlsx", ".xlsm"):
          import openpyxl
          wb = openpyxl.load_workbook(path, data_only=True)
          sections = []
          for ws in wb.worksheets:
              sections.append(_sheet_openpyxl(ws, max_rows_per_sheet))
          return "\n\n".join(sections)

      else:
          raise ValueError(f"Unsupported extension '{ext}'. Use .xlsx, .xls, or .xlsm.")

params_schema:
  {"params":[{"name":"path","desc":"ruta al archivo Excel (.xlsx, .xls, .xlsm)"},{"name":"max_rows_per_sheet","desc":"limite maximo de filas a procesar por sheet"}],"output":"string markdown con cada sheet como seccion H2 y sus datos en tablas"}


id: extract_frontmatter_py_core
name: extract_frontmatter
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def extract_frontmatter(content: str) -> tuple[str, dict | None]
tags: ["markdown","frontmatter","yaml","parsing"]
uses_functions: []
uses_types: []
returns: []
imports: ["re","yaml"]
test_file_path: python/functions/core/parse_markdown_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 23f664a0b4212ac611ece53c0cc43dcfe145617eabc961d3e574d5156fd9674f
description:
  Extrae YAML frontmatter (delimitado por ---) del inicio de un string markdown. Retorna el contenido sin frontmatter y el dict parseado (o None si no hay).

example:
  ```python
  content = "---\ntitle: Hello\nauthor: Alice\n---\n# Body\n"
  remaining, data = extract_frontmatter(content)
  # remaining = "# Body\n"
  # data = {"title": "Hello", "author": "Alice"}

  no_fm = "# Just markdown\n\nNo frontmatter."
  remaining, data = extract_frontmatter(no_fm)
  # remaining == no_fm
  # data is None
  ```

tests:
  ["contenido con frontmatter","sin frontmatter retorna None","frontmatter vacio","frontmatter con listas"]

notes:
  Funcion pura. Usa `yaml.safe_load` si PyYAML esta disponible; si no, cae back a un parser simple de `key: value`. Solo reconoce frontmatter al inicio estricto del string (posicion 0). El bloque debe estar delimitado por `---\n` de apertura y `\n---\n` de cierre.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"content","desc":"string con contenido markdown potencialmente precedido por YAML frontmatter"}],"output":"tupla (contenido_sin_frontmatter, dict_parseado_o_None)"}


id: extract_json_from_llm_py_core
name: extract_json_from_llm
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def extract_json_from_llm(content: str) -> dict
description: Extrae y parsea JSON de respuestas LLM. Maneja bloques ```json, trailing commas, None->null.
tags: ["json","llm","parsing","extraction"]
uses_functions: []
uses_types: []
returns: []
imports: ["json"]
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: f40b8c6c0368e47a5e6714394d5177cc0090d6d8099b8b81aa1526021c3145fd
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  raw = '```json\n{"key": "value", "items": [1, 2, 3,]}\n```'
  result = extract_json_from_llm(raw)
  # {"key": "value", "items": [1, 2, 3]}
  ```

notes:
  Funcion pura. Maneja errores comunes de LLMs: trailing commas, `None` en lugar de `null`, whitespace extra. Retorna dict vacio si el JSON es irrecuperable.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"content","desc":"string con respuesta LLM que contiene JSON (puede estar en bloque ```json)"}],"output":"dict parseado desde el JSON; dict vacio si no se puede recuperar"}


id: extract_markdown_headers_py_core
name: extract_markdown_headers
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def extract_markdown_headers(markdown_content: str) -> tuple[list[dict], list[str]]
description: Extrae todos los headers (h1-h6) de markdown con nivel y numero de linea, ignorando code blocks.
tags: ["markdown","parsing","headers","extraction"]
uses_functions: []
uses_types: []
returns: []
imports: ["re"]
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: c5aba41a9019c311a968cb8bb6e614f7617e0d3b13702ce18afd191c45e86009
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/page_index_md.py
example:
  ```python
  md = "# Title\n\nSome text\n\n## Section\n\n```\n# not a header\n```"
  headers, lines = extract_markdown_headers(md)
  # headers = [{"title": "Title", "level": 1, "line_num": 1}, {"title": "Section", "level": 2, "line_num": 5}]
  ```

notes:
  Funcion pura. Detecta y omite bloques de codigo (triple backtick). Retorna tupla: (lista de headers, lista de lineas originales).

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"markdown_content","desc":"string con contenido markdown"}],"output":"tupla (lista_de_headers, lista_de_lineas_originales) donde headers son dicts con title/level/line_num"}


id: extract_pdf_bookmarks_py_core
name: extract_pdf_bookmarks
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def extract_pdf_bookmarks(pdf) -> list[dict]
tags: ["pdf","bookmarks","outlines","parsing","pdfplumber"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["pdfplumber"]
tests: []
file_path: python/functions/core/extract_pdf_bookmarks.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 35b3635274d868d71f7de81a97feed0c3582e3ac40eb948e13cbc7b047d623c8
description:
  Extrae la estructura de bookmarks/outlines de un PDF abierto con pdfplumber. Retorna lista de dicts con level (1-6), title y page_num.

example:
  ```python
  import pdfplumber
  from extract_pdf_bookmarks import extract_pdf_bookmarks

  with pdfplumber.open("document.pdf") as pdf:
      bookmarks = extract_pdf_bookmarks(pdf)
      for bm in bookmarks:
          print(f"{'#' * bm['level']} {bm['title']} (page {bm['page_num']})")
  ```

notes:
  Recibe un objeto `pdfplumber.PDF` ya abierto (no un path). Construye un mapping interno `objid -> page_number` desde `pdf.pages` para resolver los destinos de outline. El nivel se limita al rango [1, 6] para compatibilidad markdown. Retorna lista vacia si el PDF no tiene outlines o si `get_outlines()` falla. Impure porque accede al estado interno de un objeto PDF ya abierto.

code:
  """Extract the bookmark/outline structure from a PDF opened with pdfplumber."""

  import pdfplumber


  def extract_pdf_bookmarks(pdf: pdfplumber.PDF) -> list[dict]:
      """Extract bookmarks/outlines from an open pdfplumber PDF object.

      Args:
          pdf: An open pdfplumber.PDF object.

      Returns:
          list[dict]: List of {"level": int, "title": str, "page_num": int | None}.
                      Level is clamped to [1, 6]. Returns empty list if no outlines.
      """
      try:
          outlines = pdf.doc.get_outlines()
      except Exception:
          return []

      if not outlines:
          return []

      # Build objid -> page_number mapping
      objid_to_page: dict[int, int] = {}
      for i, page in enumerate(pdf.pages):
          try:
              obj = page.page_obj
              objid_to_page[obj.objid] = i + 1  # 1-indexed page numbers
          except Exception:
              pass

      bookmarks = []
      for item in outlines:
          try:
              level = item[0]  # integer level from get_outlines
              title = item[1]
              dest = item[2]  # destination: page object or list

              # Clamp level to [1, 6]
              level = max(1, min(6, level))

              # Resolve destination to page number
              page_num = None
              if dest is not None:
                  if isinstance(dest, list) and len(dest) > 0:
                      # dest[0] is the page object
                      page_obj = dest[0]
                      try:
                          page_num = objid_to_page.get(page_obj.objid)
                      except Exception:
                          pass
                  else:
                      try:
                          page_num = objid_to_page.get(dest.objid)
                      except Exception:
                          pass

              bookmarks.append({"level": level, "title": str(title), "page_num": page_num})
          except Exception:
              continue

      return bookmarks

params_schema:
  {"params":[{"name":"pdf","desc":"objeto pdfplumber.PDF abierto"}],"output":"lista de dicts con campos level (1-6), title y page_num para cada bookmark"}


id: extract_pdf_text_py_core
name: extract_pdf_text
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def extract_pdf_text(pdf_path: str) -> str
description: Extrae todo el texto de un PDF concatenando todas las paginas. Usa PyPDF2.
tags: ["pdf","text","extraction","parsing"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["PyPDF2"]
example:
  ```python
  text = extract_pdf_text("/path/to/document.pdf")
  print(len(text)) # total characters
  ```

tests: []
file_path: python/functions/core/extract_pdf_text.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:44:15Z
props: []
emits: []
variant: []
content_hash: 8e77a771fd9414fc17a034dfd24eda926e20a0be55ca9269c3490002826970e2
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
notes:
  Requiere `pip install PyPDF2`. Extraccion basica de texto — no maneja OCR ni PDFs escaneados. Para PDFs complejos considerar PyMuPDF.

code:
  """Extract all text from a PDF file using PyPDF2."""

  import PyPDF2


  def extract_pdf_text(pdf_path: str) -> str:
      """Extract all text from a PDF file.

      Args:
          pdf_path: Path to the PDF file.

      Returns:
          str: Concatenated text from all pages.
      """
      pdf_reader = PyPDF2.PdfReader(pdf_path)
      text = ""
      for page in pdf_reader.pages:
          text += page.extract_text() or ""
      return text

params_schema:
  {"params":[{"name":"pdf_path","desc":"ruta al archivo PDF desde el cual extraer el texto"}],"output":"string: texto concatenado de todas las páginas del PDF"}


id: extract_text_from_file_py_core
name: extract_text_from_file
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: extract_text_from_file(file_path: str) -> str
tags: ["text","pdf","markdown","txt","encoding","extraction","file","io"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["os","fitz (PyMuPDF)","charset_normalizer","chardet"]
test_file_path: python/functions/core/extract_text_from_file_test.py
file_path: python/functions/core/extract_text_from_file.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 87cbf8b50c5dbf344779540ab951ce225c2f924da6516f26579e36fc096f2fc6
description:
  Extrae texto plano de un archivo. Soporta PDF (PyMuPDF), Markdown y TXT con deteccion automatica de encoding.

example:
  ```python
  # PDF
  text = extract_text_from_file("report.pdf")

  # Markdown
  text = extract_text_from_file("README.md")

  # TXT con encoding desconocido
  text = extract_text_from_file("notes.txt")
  ```

tests:
  ["PDF con texto extrae contenido correctamente","archivo MD UTF-8 retorna contenido","archivo TXT latin-1 detecta encoding","archivo inexistente lanza FileNotFoundError","extension no soportada lanza ValueError"]

notes:
  Para PDF usa PyMuPDF (`fitz`) que produce mejor texto que PyPDF2, especialmente en PDFs con columnas o layout complejo. Las paginas se unen con `\n\n`.

  La deteccion de encoding para archivos de texto sigue este orden de prioridad:
  1. Intenta UTF-8 directamente
  2. `charset_normalizer.from_bytes().best().encoding`
  3. `chardet.detect(data)["encoding"]`
  4. UTF-8 con `errors='replace'` como ultimo recurso

  Diferencia con `extract_pdf_text_py_core`: esa funcion usa PyPDF2 y solo soporta PDF. Esta funcion usa PyMuPDF y soporta ademas MD y TXT con deteccion de encoding.

code:
  """Extract plain text from PDF, Markdown, or TXT files."""


  SUPPORTED_EXTENSIONS = {".pdf", ".md", ".markdown", ".txt"}


  def _detect_encoding(data: bytes) -> str:
      """Detect encoding of raw bytes using multiple fallback strategies."""
      # Strategy 1: UTF-8
      try:
          data.decode("utf-8")
          return "utf-8"
      except UnicodeDecodeError:
          pass

      # Strategy 2: charset_normalizer
      try:
          from charset_normalizer import from_bytes

          result = from_bytes(data).best()
          if result is not None and result.encoding:
              return result.encoding
      except ImportError:
          pass

      # Strategy 3: chardet
      try:
          import chardet

          detected = chardet.detect(data)
          if detected and detected.get("encoding"):
              return detected["encoding"]
      except ImportError:
          pass

      # Last resort: UTF-8 with replacement
      return "utf-8"


  def extract_text_from_file(file_path: str) -> str:
      """Extract plain text from a file. Supports PDF, Markdown and TXT.

      For PDF files uses PyMuPDF (fitz) to extract text from each page,
      joining them with double newlines. For text-based files (.md, .markdown,
      .txt) reads the file with automatic encoding detection.

      Args:
          file_path: Absolute or relative path to the file.

      Returns:
          str: Extracted plain text content.

      Raises:
          FileNotFoundError: If the file does not exist.
          ValueError: If the file extension is not supported.
          ImportError: If PyMuPDF is not installed and a PDF is provided.
      """
      import os

      if not os.path.exists(file_path):
          raise FileNotFoundError(f"File not found: {file_path}")

      _, ext = os.path.splitext(file_path.lower())

      if ext == ".pdf":
          try:
              import fitz  # PyMuPDF
          except ImportError as e:
              raise ImportError(
                  "PyMuPDF is required for PDF extraction. "
                  "Install it with: pip install PyMuPDF"
              ) from e

          doc = fitz.open(file_path)
          pages = [page.get_text() for page in doc]
          return "\n\n".join(pages)

      elif ext in {".md", ".markdown", ".txt"}:
          with open(file_path, "rb") as f:
              raw = f.read()

          encoding = _detect_encoding(raw)
          try:
              return raw.decode(encoding)
          except (UnicodeDecodeError, LookupError):
              return raw.decode("utf-8", errors="replace")

      else:
          raise ValueError(
              f"Unsupported file extension: '{ext}'. "
              f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
          )

params_schema:
  {"params":[{"name":"file_path","desc":"ruta al archivo (PDF, MD o TXT)"}],"output":"string con texto extraido del archivo"}


id: fetch_and_parse_url_py_core
name: fetch_and_parse_url
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: fetch_and_parse_url(url: str, timeout: float = 30.0) -> str
tags: ["http","fetch","html","markdown","parse","url","scraping"]
uses_functions: ["detect_url_type_py_core","html_to_markdown_py_core"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["httpx"]
tests: []
file_path: python/functions/core/fetch_and_parse_url.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: c8329d93009d3c57222061e9f3da03e8378979c2d36c04c65d630fef7de47ba7
description:
  Descarga una pagina web y la convierte a markdown. Combina detect_url_type + fetch HTML + html_to_markdown en una sola operacion.

example:
  ```python
  from core.fetch_and_parse_url import fetch_and_parse_url

  # Descargar y convertir una pagina web
  md = fetch_and_parse_url("https://example.com")
  print(md)

  # Con timeout personalizado
  md = fetch_and_parse_url("https://en.wikipedia.org/wiki/Python", timeout=15.0)
  ```

notes:
  Algoritmo:
  1. `detect_url_type(url)` determina el tipo de contenido (por patron, extension o HEAD request).
  2. Si es `code_repository` → lanza Exception (requiere git clone, no HTTP fetch).
  3. Si es `pdf` → lanza Exception (requiere pdfminer/pypdf, no incluido).
  4. `httpx.get(url)` descarga el contenido con follow_redirects.
  5. Si es `webpage` o Content-Type HTML → `html_to_markdown(raw_html)`.
  6. Si es `markdown`, `text` o codigo → retorna el texto directamente.

  Lanza `Exception` con mensaje descriptivo en cualquier fallo de red o tipo no soportado.

  Funcion impura: hace I/O (HTTP requests).

code:
  """Descarga una pagina web y la convierte a markdown."""

  from __future__ import annotations


  def fetch_and_parse_url(url: str, timeout: float = 30.0) -> str:
      """Descarga una pagina web y la convierte a markdown.

      Detecta el tipo de URL con detect_url_type, descarga el contenido con
      httpx y lo convierte al formato apropiado:
      - webpage: fetch HTML → html_to_markdown
      - markdown: retorna el texto directamente
      - text/code: retorna el texto directamente
      - pdf: retorna stub (requiere dependencia externa)
      - code_repository: retorna stub (requiere clonar repo)

      Args:
          url: URL a descargar y parsear.
          timeout: Timeout en segundos para las peticiones HTTP.

      Returns:
          Contenido de la URL en formato markdown.

      Raises:
          Exception: Si falla la descarga (timeout, DNS, HTTP error) o el tipo
                     de URL no es soportado.
      """
      import httpx

      from detect_url_type import detect_url_type
      from html_to_markdown import html_to_markdown

      # Detectar tipo de URL (puede hacer HEAD request)
      url_type, _meta = detect_url_type(url, timeout=timeout)

      if url_type == "code_repository":
          raise Exception(
              f"fetch_and_parse_url: code_repository URLs require git clone, not supported. url={url!r}"
          )

      if url_type == "pdf":
          raise Exception(
              f"fetch_and_parse_url: PDF parsing requires external dependency (pdfminer/pypdf). url={url!r}"
          )

      # Fetch content via GET
      try:
          response = httpx.get(url, timeout=timeout, follow_redirects=True)
          response.raise_for_status()
      except httpx.HTTPStatusError as exc:
          raise Exception(
              f"fetch_and_parse_url: HTTP {exc.response.status_code} for {url!r}"
          ) from exc
      except Exception as exc:
          raise Exception(f"fetch_and_parse_url: request failed for {url!r}: {exc}") from exc

      content_type = response.headers.get("content-type", "").lower()
      raw_text = response.text

      if url_type == "webpage" or "text/html" in content_type:
          return html_to_markdown(raw_text)

      # markdown, text, or code files — return as-is
      return raw_text

params_schema:
  {"params":[{"name":"url","desc":"URL a descargar y convertir"},{"name":"timeout","desc":"timeout en segundos para el HTTP request"}],"output":"string markdown con el contenido convertido desde la pagina web"}


id: filter_list_py_core
name: filter_list
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def filter_list(xs: list, pred: callable) -> list
description: Filtra una lista aplicando un predicado sin mutar la original.
tags: ["list","functional","filter","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  evens = filter_list([1, 2, 3, 4], lambda n: n % 2 == 0)
  # [2, 4]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. No muta la lista original. Equivalente a `[x for x in xs if pred(x)]`.
content_hash: 300f8efb03f8a888b2abd1d68657254a40e50467089c1010b72318bbb0a8bbd9
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista a filtrar"},{"name":"pred","desc":"predicado (funcion que retorna bool) para seleccionar elementos"}],"output":"lista con solo los elementos que cumplen el predicado"}


id: find_py_core
name: find
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def find(xs: list, pred: callable)
description: Encuentra el primer elemento que cumple el predicado. Retorna None si no hay coincidencia.
tags: ["list","functional","find","search","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = find([1, 2, 3, 4], lambda n: n > 2)
  # 3
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Retorna None si ningun elemento cumple el predicado. Cortocircuita al primer match.
content_hash: 522cafc65271b3d851763f91df8a0aedc268b5926a01acbdfa1483899be17a60
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista donde buscar"},{"name":"pred","desc":"predicado (funcion que retorna bool) para encontrar un elemento"}],"output":"primer elemento que cumple el predicado, o None si no hay coincidencia"}


id: find_headings_py_core
name: find_headings
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def find_headings(content: str) -> list[tuple[int, int, str, int]]
tags: ["markdown","headings","parsing","extraction"]
uses_functions: []
uses_types: []
returns: []
imports: ["re"]
test_file_path: python/functions/core/parse_markdown_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 05c40c3c4b0071820d34026166eb0b64c60d7411997671bc527e79cb487e6c39
description:
  Encuentra todos los headings markdown (# a ######), excluyendo los que estan dentro de code blocks, HTML comments y bloques indentados. Retorna lista de (start_pos, end_pos, title, level).

example:
  ```python
  content = "# Title\n\nSome text\n\n## Section\n\n```\n# Ignored\n```\n"
  headings = find_headings(content)
  # [(0, 7, "Title", 1), (22, 33, "Section", 2)]
  # (positions approximated)
  ```

tests:
  ["headings normales detectados correctamente","headings dentro de code blocks no detectados","headings escapados ignorados","headings en HTML comments ignorados"]

notes:
  Funcion pura. Excluye tres tipos de contextos: bloques de codigo triple backtick, comentarios HTML (`<!-- ... -->`), y lineas indentadas con 4 espacios o tabulacion. Tambien filtra headings precedidos por backslash (`\#`). Diferencia clave respecto a `extract_markdown_headers`: esta funcion retorna posiciones de caracter, no numeros de linea, lo que facilita la extraccion de contenido entre headings.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"content","desc":"string con contenido markdown"}],"output":"lista de tuplas (start_pos, end_pos, title, level) para cada heading detectado"}


id: find_index_py_core
name: find_index
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def find_index(xs: list, pred: callable) -> int
description: Encuentra el indice del primer elemento que cumple el predicado. Retorna -1 si no hay coincidencia.
tags: ["list","functional","find","index","search","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  idx = find_index([10, 20, 30, 40], lambda n: n > 25)
  # 2
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Retorna -1 si ningun elemento cumple el predicado.
content_hash: ded9f0432c6d2bed815b60acc4b1d175ff041cb3533f18318bf0210cf7700c75
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista donde buscar"},{"name":"pred","desc":"predicado (funcion que retorna bool) para encontrar un indice"}],"output":"entero con el indice del primer elemento que cumple, o -1 si no hay"}


id: flat_map_py_core
name: flat_map
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def flat_map(xs: list, fn: callable) -> list
description: Aplica una funcion que retorna listas a cada elemento y aplana el resultado un nivel.
tags: ["list","functional","flatmap","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = flat_map([1, 2, 3], lambda n: [n, n * 10])
  # [1, 10, 2, 20, 3, 30]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Equivalente a flatten(map_list(xs, fn)). Solo aplana un nivel.
content_hash: 813c024a0699cd37db249749bf43ab15f1d0b6bd784988e108f2a8f79f118bfd
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista a transformar"},{"name":"fn","desc":"funcion que recibe un elemento y retorna una lista"}],"output":"lista plana resultado de aplicar fn a cada elemento y aplanar un nivel"}


id: flatten_py_core
name: flatten
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def flatten(xss: list) -> list
description: Aplana una lista de listas un nivel, concatenando las sublistas.
tags: ["list","functional","flatten","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  result = flatten([[1, 2], [3], [4, 5]])
  # [1, 2, 3, 4, 5]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Solo aplana un nivel de anidamiento.
content_hash: 96915997f8c298382e1f9228cfb197156b1efd52d1c8a6275b6daa688ffe7f1c
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xss","desc":"lista de listas a aplanar"}],"output":"lista plana con todos los elementos concatenados un nivel"}


id: flatten_tree_py_core
name: flatten_tree
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def flatten_tree(structure: Any) -> list[dict]
description: Aplana un arbol jerarquico (dict con 'nodes') a lista plana sin hijos. Deep copy de cada nodo.
tags: ["tree","flatten","hierarchy","functional"]
uses_functions: []
uses_types: []
returns: []
imports: ["copy"]
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 94e862a8db22802dce148e40532f04c1acdb9b2cd3bb9c6384f2ae7ecea299a9
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  tree = [{"title": "A", "nodes": [{"title": "A1", "nodes": []}]}]
  flatten_tree(tree)
  # [{"title": "A"}, {"title": "A1"}]
  ```

notes:
  Funcion pura. Usa deep copy para no mutar el arbol original. Elimina el campo 'nodes' de cada nodo aplanado.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"structure","desc":"arbol jerarquico (dict con campo 'nodes' anidado recursivamente)"}],"output":"lista plana de dicts sin campo 'nodes', deep copy de cada nodo"}


id: format_iso8601_py_core
name: format_iso8601
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: format_iso8601(dt: datetime) -> str
tags: ["datetime","iso8601","format","time","utc"]
uses_functions: []
uses_types: []
returns: []
imports: ["datetime"]
test_file_path: python/functions/core/format_iso8601_test.py
file_path: python/functions/core/format_iso8601.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 762c1f38f78a8ac2d49074a18bafef87612af53ead0cbe49b48e915336566453
description:
  Formatea un datetime a ISO 8601 UTC con milisegundos. Formato: yyyy-MM-ddTHH:mm:ss.SSSZ. Si naive asume UTC, si aware convierte a UTC.

example:
  ```python
  from datetime import datetime, timezone, timedelta
  from format_iso8601 import format_iso8601

  # Naive (asume UTC)
  s = format_iso8601(datetime(2026, 2, 21, 13, 20, 23, 147000))
  # "2026-02-21T13:20:23.147Z"

  # Con timezone +8
  tz8 = timezone(timedelta(hours=8))
  s = format_iso8601(datetime(2026, 2, 21, 21, 20, 23, 147000, tzinfo=tz8))
  # "2026-02-21T13:20:23.147Z"
  ```

tests:
  ["datetime naive formateado como UTC","datetime con timezone convertido a UTC","datetime UTC sin conversion"]

notes:
  Algoritmo:
  1. Si naive: `dt.replace(tzinfo=timezone.utc)`.
  2. Si aware: `dt.astimezone(timezone.utc)`.
  3. `dt.isoformat(timespec="milliseconds").replace("+00:00", "Z")`.

  Funcion pura. No hace I/O ni tiene efectos secundarios.

code:
  """Formatea un datetime a ISO 8601 UTC con milisegundos."""

  from datetime import datetime, timezone


  def format_iso8601(dt: datetime) -> str:
      """Formatea un datetime a ISO 8601 UTC con milisegundos.

      Formato de salida: ``yyyy-MM-ddTHH:mm:ss.SSSZ``

      Si el datetime es naive (sin tzinfo), se asume UTC.
      Si el datetime es aware, se convierte a UTC antes de formatear.

      Args:
          dt: datetime a formatear. Puede ser naive o aware.

      Returns:
          String ISO 8601 en UTC con milisegundos, terminando en 'Z'.
      """
      if dt.tzinfo is None:
          dt = dt.replace(tzinfo=timezone.utc)
      else:
          dt = dt.astimezone(timezone.utc)
      return dt.isoformat(timespec="milliseconds").replace("+00:00", "Z")

params_schema:
  {"params":[{"name":"dt","desc":"datetime object (puede ser naive o aware)"}],"output":"string ISO 8601 UTC con milisegundos formato yyyy-MM-ddTHH:mm:ss.SSSZ"}


id: format_simplified_py_core
name: format_simplified
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: format_simplified(dt: datetime, now: datetime) -> str
tags: ["datetime","format","time","human","display"]
uses_functions: []
uses_types: []
returns: []
imports: ["datetime"]
test_file_path: python/functions/core/format_simplified_test.py
file_path: python/functions/core/format_simplified.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: e3606391d5246c178478ba865432a70864a7dc0bf768d82c44bc7efc4ec822cf
description:
  Formato humano simplificado: si dt es del mismo dia que now muestra HH:MM:SS, si no muestra YYYY-MM-DD.

example:
  ```python
  from datetime import datetime
  from format_simplified import format_simplified

  now = datetime(2026, 2, 21, 15, 0, 0)

  # Mismo dia
  s = format_simplified(datetime(2026, 2, 21, 9, 30, 0), now)
  # "09:30:00"

  # Dia anterior
  s = format_simplified(datetime(2026, 2, 20, 9, 30, 0), now)
  # "2026-02-20"
  ```

tests:
  ["mismo dia muestra formato hora","dia anterior muestra formato fecha","exactamente 24h muestra formato fecha"]

notes:
  Algoritmo:
  1. Remover tzinfo de ambos datetimes para comparacion simple (`replace(tzinfo=None)`).
  2. Si `(now - dt).days < 1`: retornar `dt.strftime("%H:%M:%S")`.
  3. Si no: retornar `dt.strftime("%Y-%m-%d")`.

  El umbral de 1 dia en `timedelta.days` significa que cualquier diferencia
  menor a 24 horas se muestra como hora. Un dt exactamente 24h atras
  tendra `days == 1`, mostrando fecha.

  Funcion pura. No hace I/O ni tiene efectos secundarios.

code:
  """Formato humano simplificado de datetime: hora si es hoy, fecha si es otro dia."""

  from datetime import datetime


  def format_simplified(dt: datetime, now: datetime) -> str:
      """Formato humano simplificado de datetime.

      Si ``dt`` es del mismo dia que ``now`` (diferencia < 1 dia), retorna
      la hora en formato ``HH:MM:SS``. En caso contrario retorna la fecha
      en formato ``YYYY-MM-DD``.

      Args:
          dt: datetime a formatear.
          now: datetime de referencia (el momento actual).

      Returns:
          String ``HH:MM:SS`` si mismo dia, ``YYYY-MM-DD`` si otro dia.
      """
      dt_naive = dt.replace(tzinfo=None)
      now_naive = now.replace(tzinfo=None)
      diff = now_naive - dt_naive
      if diff.days < 1:
          return dt.strftime("%H:%M:%S")
      return dt.strftime("%Y-%m-%d")

params_schema:
  {"params":[{"name":"dt","desc":"datetime a formatear"},{"name":"now","desc":"datetime actual para comparacion"}],"output":"string con formato humano simplificado: HH:MM:SS si mismo dia, YYYY-MM-DD si otro dia"}


id: format_table_to_markdown_py_core
name: format_table_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def format_table_to_markdown(rows: list[list[str]], has_header: bool = True) -> str
tags: ["markdown","table","formatting","text","pure"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/format_table_to_markdown_test.py
file_path: python/functions/core/format_table_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: f00dd3baebc5a88bd9be888e6982124f18622ead3ae5bf1f590ade99f294c686
description:
  Convierte una lista 2D de celdas a tabla markdown con alineacion de columnas. Escapa pipes en celdas y añade separador header.

example:
  ```python
  rows = [["Name", "Age"], ["Alice", "30"], ["Bob", "25"]]
  md = format_table_to_markdown(rows)
  # | Name  | Age |
  # | ----- | --- |
  # | Alice | 30  |
  # | Bob   | 25  |
  ```

tests:
  ["tabla normal","tabla con celdas vacias","tabla con 1 fila","tabla vacia","celdas con pipes","sin header"]

notes:
  Funcion pura. No tiene dependencias externas. Calcula el ancho maximo por columna para alinear. El separador usa minimo 3 guiones por columna para cumplir con la especificacion markdown. Escapa los pipes dentro de celdas con `\|`. Si `has_header=False`, omite la fila separadora.

code:
  """Convert a 2D list of cells to a markdown table with column alignment."""


  def format_table_to_markdown(rows: list[list[str]], has_header: bool = True) -> str:
      """Convert a 2D list of cells to a markdown table.

      Args:
          rows: 2D list where each inner list is a row of cell strings.
          has_header: If True, the first row is treated as the header row.

      Returns:
          str: Markdown table string. Returns empty string for empty input.
      """
      if not rows:
          return ""

      def escape_cell(cell: str) -> str:
          return str(cell).replace("|", "\\|")

      # Determine column count from widest row
      col_count = max(len(row) for row in rows)

      # Pad rows to same column count
      padded = [row + [""] * (col_count - len(row)) for row in rows]

      # Escape pipe characters in all cells
      escaped = [[escape_cell(cell) for cell in row] for row in padded]

      # Calculate max width per column
      col_widths = [
          max(len(escaped[r][c]) for r in range(len(escaped)))
          for c in range(col_count)
      ]
      col_widths = [max(w, 3) for w in col_widths]  # minimum width of 3 for separator

      def format_row(row: list[str]) -> str:
          cells = [cell.ljust(col_widths[i]) for i, cell in enumerate(row)]
          return "| " + " | ".join(cells) + " |"

      lines = []

      if has_header and len(escaped) >= 1:
          lines.append(format_row(escaped[0]))
          separator = "| " + " | ".join("-" * col_widths[i] for i in range(col_count)) + " |"
          lines.append(separator)
          for row in escaped[1:]:
              lines.append(format_row(row))
      else:
          for row in escaped:
              lines.append(format_row(row))

      return "\n".join(lines)

params_schema:
  {"params":[{"name":"rows","desc":"lista 2D de listas con celdas (cada fila es una lista de strings)"},{"name":"has_header","desc":"si True la primera fila se trata como header y se anade separador"}],"output":"string con tabla markdown formateada con pipes y alineacion de columnas"}


id: format_tree_structure_py_core
name: format_tree_structure
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def format_tree_structure(structure: Any, order: list[str] = None) -> Any
description: Reordena campos de cada nodo de un arbol segun orden de claves especificado.
tags: ["tree","format","order","structure"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 75eccec381c981f23d487e35894b3a3d56c4c66d856b9e187d3ec2d2d6843b4f
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  tree = [{"text": "...", "title": "Intro", "node_id": "0001"}]
  format_tree_structure(tree, order=["title", "node_id", "text"])
  # [{"title": "Intro", "node_id": "0001", "text": "..."}]
  ```

notes:
  Funcion pura. Elimina nodos vacios (nodes=[]) automaticamente. Claves no listadas en order se descartan.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"structure","desc":"arbol jerarquico (dict con campo 'nodes' anidado recursivamente)"},{"name":"order","desc":"lista de nombres de campos en el orden deseado para reordenar"}],"output":"estructura arbol con los campos de cada nodo reordenados segun order"}


id: from_csv_py_core
name: from_csv
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: from_csv(text: str, delimiter: str = ',', has_header: bool = True) -> list[dict]
tags: ["csv","parser","import","tabular","format"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/from_csv_test.py
file_path: python/functions/core/from_csv.py
created_at: 2026-04-05T11:25:56Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 04293f5f76f14588e0aebe756ea36a85d0c28c2a2085228387190545c3531062
description:
  Parser CSV a datos tabulares. Complemento de to_csv. Soporta campos entre comillas con escaping RFC 4180. Si has_header=False, genera keys col_0, col_1, etc.

example:
  ```python
  text = "nombre,edad\r\nAna,30\r\nBob,25"
  rows = from_csv(text)
  # [{"nombre": "Ana", "edad": "30"}, {"nombre": "Bob", "edad": "25"}]

  # Sin header
  text = "Ana,30\nBob,25"
  rows = from_csv(text, has_header=False)
  # [{"col_0": "Ana", "col_1": "30"}, {"col_0": "Bob", "col_1": "25"}]

  # Con escaping
  text = 'a,b\r\n"dijo ""hola""","uno,dos"'
  rows = from_csv(text)
  # [{"a": 'dijo "hola"', "b": "uno,dos"}]
  ```

tests:
  ["csv simple con header","campos con escaping","sin header keys generadas","lineas vacias ignoradas","un solo campo por fila"]

notes:
  Parser manual sin el modulo csv de stdlib. Normaliza CRLF y LF antes de procesar.
  Ignora lineas vacias. Todos los valores son strings — la conversion de tipos queda a cargo del caller.

code:
  """Parser CSV a datos tabulares (RFC 4180). Complemento de to_csv."""


  def _parse_row(line: str, delimiter: str) -> list[str]:
      """Parsea una linea CSV respetando campos entre comillas (RFC 4180)."""
      fields: list[str] = []
      field_chars: list[str] = []
      in_quotes = False
      i = 0

      while i < len(line):
          ch = line[i]

          if in_quotes:
              if ch == '"':
                  # Comilla doble escapada o cierre de campo
                  if i + 1 < len(line) and line[i + 1] == '"':
                      field_chars.append('"')
                      i += 2
                      continue
                  else:
                      in_quotes = False
              else:
                  field_chars.append(ch)
          else:
              if ch == '"' and not field_chars:
                  in_quotes = True
              elif ch == delimiter:
                  fields.append("".join(field_chars))
                  field_chars = []
              else:
                  field_chars.append(ch)
          i += 1

      fields.append("".join(field_chars))
      return fields


  def from_csv(
      text: str,
      delimiter: str = ",",
      has_header: bool = True,
  ) -> list[dict]:
      """Parser CSV a lista de dicts.

      Complemento de to_csv. Soporta campos entre comillas con escaping RFC 4180.
      Si has_header=False, genera keys col_0, col_1, etc.

      Args:
          text: Contenido CSV completo como string.
          delimiter: Separador de campos. Por defecto coma.
          has_header: Si True, primera fila es el encabezado.
                      Si False, genera keys col_0, col_1, ...

      Returns:
          Lista de dicts. Lista vacia si el texto esta vacio o solo tiene header.
      """
      # Normalizar line endings
      normalized = text.replace("\r\n", "\n").replace("\r", "\n")
      lines = [l for l in normalized.split("\n") if l.strip() != ""]

      if not lines:
          return []

      if has_header:
          headers = _parse_row(lines[0], delimiter)
          data_lines = lines[1:]
      else:
          # Determinar numero de columnas desde la primera fila
          sample = _parse_row(lines[0], delimiter)
          headers = [f"col_{i}" for i in range(len(sample))]
          data_lines = lines

      result: list[dict] = []
      for line in data_lines:
          fields = _parse_row(line, delimiter)
          # Alinear con headers (rellenar con "" si faltan campos)
          row = {}
          for i, header in enumerate(headers):
              row[header] = fields[i] if i < len(fields) else ""
          result.append(row)

      return result

params_schema:
  {"params":[{"name":"text","desc":"string con contenido CSV"},{"name":"delimiter","desc":"caracter delimitador de campos (default coma)"},{"name":"has_header","desc":"si True la primera fila se usa como nombres de columnas"}],"output":"lista de dicts donde cada dict es una fila y las keys son los nombres de columnas"}


id: from_jsonl_py_core
name: from_jsonl
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: from_jsonl(text: str) -> list[dict]
tags: ["jsonl","json","parser","import","streaming","format"]
uses_functions: []
uses_types: []
returns: []
imports: ["json"]
tests: ["jsonl valido","lineas vacias intercaladas","linea invalida raise con numero"]
test_file_path: python/functions/core/from_jsonl_test.py
file_path: python/functions/core/from_jsonl.py
created_at: 2026-04-05T14:43:57Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 43f44c7feaebc55cbc025f41e43cad1d9b518867fb2a4f4338721de272b19b60
description:
  Parser JSONL a lista de dicts. Ignora lineas vacias. Lanza ValueError con el numero de linea si una linea contiene JSON invalido. Complemento de to_jsonl.

example:
  ```python
  text = '{"id": 1}\n{"id": 2}'
  rows = from_jsonl(text)
  # [{"id": 1}, {"id": 2}]

  # Lineas vacias ignoradas
  text = '{"id": 1}\n\n{"id": 2}\n'
  rows = from_jsonl(text)
  # [{"id": 1}, {"id": 2}]

  # JSON invalido levanta error con numero de linea
  try:
      from_jsonl('{"ok": 1}\nnot-json')
  except ValueError as e:
      print(e)  # "JSON invalido en linea 2: ..."
  ```

notes:
  Aunque se declara pure (no hace I/O), puede lanzar ValueError para JSON invalido.
  Esto es consistente con la convencion del registry: funciones puras pueden lanzar
  excepciones de validacion — solo las funciones impuras retornan error como valor.

code:
  """Parser JSON Lines (JSONL) a lista de dicts. Complemento de to_jsonl."""

  import json


  def from_jsonl(text: str) -> list[dict]:
      """Parser JSONL a lista de dicts.

      Complemento de to_jsonl. Ignora lineas vacias. Lanza ValueError si
      una linea contiene JSON invalido, indicando el numero de linea.

      Args:
          text: Contenido JSONL como string (una linea JSON por linea).

      Returns:
          Lista de dicts parseados.

      Raises:
          ValueError: Si una linea no es JSON valido, con el numero de linea.
      """
      result: list[dict] = []

      for line_num, line in enumerate(text.splitlines(), start=1):
          stripped = line.strip()
          if not stripped:
              continue
          try:
              parsed = json.loads(stripped)
          except json.JSONDecodeError as exc:
              raise ValueError(
                  f"JSON invalido en linea {line_num}: {exc}"
              ) from exc
          result.append(parsed)

      return result

params_schema:
  {"params":[{"name":"text","desc":"string con contenido JSONL (JSON Lines, una linea por objeto)"}],"output":"lista de dicts, cada uno parseado desde una linea del JSONL"}


id: generate_html_report_py_core
name: generate_html_report
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: generate_html_report(title: str, sections: list[dict]) -> str
tags: ["html","report","export","table","kpi","template","format"]
uses_functions: []
uses_types: []
returns: []
imports: ["re"]
test_file_path: python/functions/core/generate_html_report_test.py
file_path: python/functions/core/generate_html_report.py
created_at: 2026-04-05T14:43:57Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 3307f5c9483a50c364d26b495d54d466f6d56f3744836040ebda429bf4169f81
description:
  Genera un reporte HTML autocontenido con CSS inline. Soporta secciones de tipo table (list[dict]), text (str con markdown basico), kpi (cards con label/value/delta) y list (list[str]). Para exportar resultados de pipelines sin servidor.

example:
  ```python
  sections = [
      {
          "heading": "Resumen ejecutivo",
          "type": "kpi",
          "data": [
              {"label": "Revenue", "value": "$1.2M", "delta": "+15%"},
              {"label": "Churn", "value": "3.2%", "delta": "-0.5%"},
          ],
      },
      {
          "heading": "Top usuarios",
          "type": "table",
          "data": [
              {"usuario": "ana@example.com", "compras": 42},
              {"usuario": "bob@example.com", "compras": 38},
          ],
      },
      {
          "heading": "Notas",
          "type": "text",
          "data": "Datos del **trimestre Q1**. Ver [dashboard](https://example.com).",
      },
  ]

  html = generate_html_report("Reporte Mensual", sections)
  # Retorna string HTML completo con DOCTYPE, head con CSS inline, body con secciones
  ```

tests:
  ["reporte con una tabla","reporte con multiples secciones mixtas","kpi con deltas positivos y negativos","caracteres especiales html escapados en data","titulo con caracteres especiales"]

notes:
  CSS completamente inline en `<style>`. Tema minimalista con max-width 960px, sans-serif,
  tabla con zebra stripes, cards KPI con colores verde/rojo para deltas.
  Todo el contenido del usuario pasa por HTML escape para proteger contra XSS.

documentation:
  ## Tipos de seccion

  - **table**: `data` es `list[dict]` — renderiza `<table>` con headers extraidos de las keys
  - **text**: `data` es `str` — soporta `**bold**` y `[text](url)`, escapa HTML
  - **kpi**: `data` es `list[{"label", "value", "delta"}]` — cards con colores para delta positivo/negativo
  - **list**: `data` es `list[str]` — renderiza `<ul><li>...</li></ul>`

code:
  """Genera reportes HTML autocontenidos con CSS inline."""


  _HTML_ESCAPES = {
      "&": "&amp;",
      "<": "&lt;",
      ">": "&gt;",
      '"': "&quot;",
      "'": "&#x27;",
  }


  def _esc(value: str) -> str:
      for ch, entity in _HTML_ESCAPES.items():
          value = value.replace(ch, entity)
      return value


  def _render_table(data: list[dict]) -> str:
      if not data:
          return "<p><em>(sin datos)</em></p>"
      headers = list(data[0].keys())
      rows_html = ""
      for i, row in enumerate(data):
          cls = ' class="zebra"' if i % 2 == 1 else ""
          cells = "".join(f"<td>{_esc(str(row.get(h, '')))}</td>" for h in headers)
          rows_html += f"<tr{cls}>{cells}</tr>\n"
      headers_html = "".join(f"<th>{_esc(h)}</th>" for h in headers)
      return (
          f"<table>\n<thead><tr>{headers_html}</tr></thead>\n"
          f"<tbody>\n{rows_html}</tbody>\n</table>"
      )


  def _render_text(data: str) -> str:
      # Markdown basico: **bold** y [text](url)
      import re

      text = _esc(str(data))
      # Bold: **text** (despues de escapar, & no interfiere)
      text = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", text)
      # Links: [text](url)
      text = re.sub(r"\[(.+?)\]\((.+?)\)", r'<a href="\2">\1</a>', text)
      return f"<p>{text}</p>"


  def _render_kpi(data: list[dict]) -> str:
      cards = ""
      for kpi in data:
          label = _esc(str(kpi.get("label", "")))
          value = _esc(str(kpi.get("value", "")))
          delta = kpi.get("delta")
          delta_html = ""
          if delta is not None:
              delta_str = str(delta)
              if delta_str.startswith("+"):
                  delta_html = f'<span class="delta-pos">{_esc(delta_str)}</span>'
              elif delta_str.startswith("-"):
                  delta_html = f'<span class="delta-neg">{_esc(delta_str)}</span>'
              else:
                  delta_html = f'<span class="delta-neutral">{_esc(delta_str)}</span>'
          cards += (
              f'<div class="kpi-card">'
              f'<div class="kpi-label">{label}</div>'
              f'<div class="kpi-value">{value}</div>'
              f"{delta_html}"
              f"</div>\n"
          )
      return f'<div class="kpi-grid">\n{cards}</div>'


  def _render_list(data: list[str]) -> str:
      items = "".join(f"<li>{_esc(str(item))}</li>\n" for item in data)
      return f"<ul>\n{items}</ul>"


  _CSS = """
  body {
      font-family: sans-serif;
      max-width: 960px;
      margin: 2rem auto;
      padding: 0 1rem;
      color: #222;
      background: #fff;
  }
  h1 { font-size: 1.8rem; border-bottom: 2px solid #ddd; padding-bottom: .5rem; }
  h2 { font-size: 1.3rem; margin-top: 2rem; color: #333; }
  table { border-collapse: collapse; width: 100%; margin: 1rem 0; font-size: .95rem; }
  th { background: #f0f0f0; text-align: left; padding: .5rem .75rem; border: 1px solid #ddd; }
  td { padding: .45rem .75rem; border: 1px solid #ddd; }
  tr.zebra { background: #f9f9f9; }
  ul { padding-left: 1.5rem; }
  li { margin: .3rem 0; }
  p { line-height: 1.6; }
  a { color: #0066cc; }
  .kpi-grid { display: flex; flex-wrap: wrap; gap: 1rem; margin: 1rem 0; }
  .kpi-card {
      border: 1px solid #ddd;
      border-radius: 6px;
      padding: 1rem 1.5rem;
      min-width: 140px;
      background: #fafafa;
  }
  .kpi-label { font-size: .85rem; color: #666; margin-bottom: .25rem; }
  .kpi-value { font-size: 1.6rem; font-weight: bold; }
  .delta-pos { color: #16a34a; font-size: .9rem; }
  .delta-neg { color: #dc2626; font-size: .9rem; }
  .delta-neutral { color: #888; font-size: .9rem; }
  """.strip()


  def generate_html_report(title: str, sections: list[dict]) -> str:
      """Genera un reporte HTML autocontenido con CSS inline.

      Cada seccion es un dict con:
          heading: str — titulo de la seccion
          type: "table" | "text" | "kpi" | "list" — tipo de contenido
          data: contenido segun el tipo:
              table -> list[dict]
              text  -> str (soporta **bold** y [links](url))
              kpi   -> list[{"label": str, "value": str|number, "delta": str|None}]
              list  -> list[str]

      No requiere servidor — todo el CSS va inline en <style>.

      Args:
          title: Titulo del reporte (mostrado en <h1> y <title>).
          sections: Lista de secciones a incluir en el reporte.

      Returns:
          String HTML completo con DOCTYPE.
      """
      sections_html = ""
      for section in sections:
          heading = _esc(str(section.get("heading", "")))
          kind = section.get("type", "text")
          data = section.get("data")

          if kind == "table":
              content = _render_table(data or [])
          elif kind == "kpi":
              content = _render_kpi(data or [])
          elif kind == "list":
              content = _render_list(data or [])
          else:
              content = _render_text(str(data or ""))

          sections_html += f"<section>\n<h2>{heading}</h2>\n{content}\n</section>\n"

      return (
          "<!DOCTYPE html>\n"
          "<html lang='es'>\n"
          "<head>\n"
          "<meta charset='UTF-8'>\n"
          "<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
          f"<title>{_esc(title)}</title>\n"
          f"<style>\n{_CSS}\n</style>\n"
          "</head>\n"
          "<body>\n"
          f"<h1>{_esc(title)}</h1>\n"
          f"{sections_html}"
          "</body>\n"
          "</html>"
      )

params_schema:
  {"params":[{"name":"title","desc":"titulo del reporte HTML"},{"name":"sections","desc":"lista de dicts con campos heading, type, data para cada seccion"}],"output":"string HTML completo autocontenido con CSS inline y todas las secciones renderizadas"}


id: generate_pwa_manifest_py_core
name: generate_pwa_manifest
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["pwa","manifest","web","json","frontend"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/generate_pwa_manifest_test.py
file_path: python/functions/core/generate_pwa_manifest.py
created_at: 2026-04-06T16:39:56Z
updated_at: 2026-04-06T16:39:56Z
props: []
emits: []
variant: []
content_hash: 610fcff3f70f6928736f99b71521f3a2b9c6436a264f275a3b245b008eb2a0e1
signature:
  def generate_pwa_manifest(name: str, short_name: str, description: str = "", start_url: str = "/", display: str = "standalone", orientation: str = "portrait", theme_color: str = "#000000", background_color: str = "#000000", icons: list[dict] | None = None, categories: list[str] | None = None, lang: str = "es") -> dict

description:
  Genera el contenido de un manifest.json para Progressive Web Apps segun el estandar W3C Web App Manifest. Retorna un dict listo para json.dumps(). Si icons es None genera placeholders estandar para 192x192 y 512x512.

example:
  ```python
  manifest = generate_pwa_manifest(
      name="Voice Guide",
      short_name="VoiceGuide",
      description="Guia por voz con POIs",
      theme_color="#12b886",
      background_color="#1a1b1e",
  )
  # {
  #   "name": "Voice Guide",
  #   "short_name": "VoiceGuide",
  #   "start_url": "/",
  #   "scope": "/",
  #   "display": "standalone",
  #   "orientation": "portrait",
  #   "theme_color": "#12b886",
  #   "background_color": "#1a1b1e",
  #   "icons": [
  #     {"src": "/icons/icon-192x192.png", "sizes": "192x192", "type": "image/png", "purpose": "any maskable"},
  #     {"src": "/icons/icon-512x512.png", "sizes": "512x512", "type": "image/png", "purpose": "any maskable"},
  #   ],
  #   "categories": [],
  #   "lang": "es",
  # }

  import json
  with open("public/manifest.json", "w") as f:
      json.dump(manifest, f, indent=2, ensure_ascii=False)
  ```

tests:
  ["valores por defecto generan manifest valido con icons placeholder","icons custom se respetan","todos los campos se incluyen correctamente","categories none produce lista vacia","json dumps compatible"]

notes:
  Funcion pura — sin I/O, sin efectos secundarios. El caller es responsable de escribir el archivo con `json.dumps()` o `json.dump()`.

  Los valores de `display` validos segun W3C son: `fullscreen`, `standalone`, `minimal-ui`, `browser`. La funcion no valida el valor — acepta cualquier string para permitir valores futuros del estandar.

  El campo `scope` siempre se fija a `"/"`. Si se necesita un scope distinto, el caller puede modificar el dict resultante antes de serializar.

code:
  """Genera el contenido de un manifest.json para Progressive Web Apps (W3C Web App Manifest)."""


  def generate_pwa_manifest(
      name: str,
      short_name: str,
      description: str = "",
      start_url: str = "/",
      display: str = "standalone",
      orientation: str = "portrait",
      theme_color: str = "#000000",
      background_color: str = "#000000",
      icons: list[dict] | None = None,
      categories: list[str] | None = None,
      lang: str = "es",
  ) -> dict:
      """Genera el contenido de un manifest.json para Progressive Web Apps.

      Args:
          name: Nombre completo de la aplicacion.
          short_name: Nombre corto para launchers y pantallas de inicio.
          description: Descripcion de la aplicacion.
          start_url: URL de inicio de la PWA.
          display: Modo de visualizacion (standalone, fullscreen, minimal-ui, browser).
          orientation: Orientacion preferida (portrait, landscape, any).
          theme_color: Color del tema en hex.
          background_color: Color de fondo en hex.
          icons: Lista de iconos con src, sizes, type y purpose. Si es None, usa placeholders estandar.
          categories: Lista de categorias de la app. Si es None, usa lista vacia.
          lang: Idioma principal de la aplicacion (BCP 47).

      Returns:
          Dict listo para json.dumps() con la estructura estandar W3C Web App Manifest.
      """
      if icons is None:
          icons = [
              {
                  "src": "/icons/icon-192x192.png",
                  "sizes": "192x192",
                  "type": "image/png",
                  "purpose": "any maskable",
              },
              {
                  "src": "/icons/icon-512x512.png",
                  "sizes": "512x512",
                  "type": "image/png",
                  "purpose": "any maskable",
              },
          ]

      if categories is None:
          categories = []

      return {
          "name": name,
          "short_name": short_name,
          "description": description,
          "start_url": start_url,
          "scope": "/",
          "display": display,
          "orientation": orientation,
          "theme_color": theme_color,
          "background_color": background_color,
          "icons": icons,
          "categories": categories,
          "lang": lang,
      }

params_schema:
  {"params":[{"name":"name","desc":"Nombre completo de la aplicacion que aparece en la pantalla de instalacion."},{"name":"short_name","desc":"Nombre corto para launchers y pantallas de inicio (maximo 12 caracteres recomendado)."},{"name":"description","desc":"Descripcion de la aplicacion. Cadena vacia por defecto."},{"name":"start_url","desc":"URL de inicio al lanzar la PWA desde la pantalla de inicio. Por defecto '/'."},{"name":"display","desc":"Modo de visualizacion: standalone, fullscreen, minimal-ui o browser. Por defecto standalone."},{"name":"orientation","desc":"Orientacion preferida de pantalla: portrait, landscape o any. Por defecto portrait."},{"name":"theme_color","desc":"Color del tema en formato hex (#RRGGBB) para la barra de estado y UI del SO. Por defecto #000000."},{"name":"background_color","desc":"Color de fondo en formato hex (#RRGGBB) mostrado durante la carga. Por defecto #000000."},{"name":"icons","desc":"Lista de dicts con src, sizes, type y purpose. Si es None genera placeholders para 192x192 y 512x512."},{"name":"categories","desc":"Lista de categorias W3C (navigation, shopping, travel, etc). Si es None usa lista vacia."},{"name":"lang","desc":"Codigo de idioma BCP 47 del contenido principal de la app. Por defecto 'es'."}],"output":"Dict con la estructura completa W3C Web App Manifest: name, short_name, description, start_url, scope, display, orientation, theme_color, background_color, icons, categories y lang."}


id: generate_service_worker_py_core
name: generate_service_worker
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["pwa","service-worker","javascript","cache","offline","codegen"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/generate_service_worker_test.py
file_path: python/functions/core/generate_service_worker.py
created_at: 2026-04-06T16:40:15Z
updated_at: 2026-04-06T16:40:15Z
props: []
emits: []
variant: []
content_hash: 7a8fb361c8b06ed2481fa4b1593057ae486e9e7e8c8fe5d07f79b9465c5c3202
signature:
  def generate_service_worker(cache_name: str = 'app-cache-v1', precache_urls: list[str] | None = None, network_first_patterns: list[str] | None = None) -> str

description:
  Genera el codigo JavaScript de un Service Worker basico para PWAs con estrategia cache-first para assets estaticos y network-first para API calls.

example:
  ```python
  sw_code = generate_service_worker(
      cache_name="voice-guide-v1",
      precache_urls=["/", "/index.html", "/manifest.json"],
      network_first_patterns=["/api/"],
  )
  # Escribir al filesystem (operacion impura, fuera de esta funcion):
  # with open("public/sw.js", "w") as f:
  #     f.write(sw_code)
  ```

tests:
  ["SW por defecto contiene install/activate/fetch handlers","Custom precache_urls aparecen en el codigo","Custom network_first_patterns aparecen en el fetch handler","El cache_name personalizado aparece en el codigo generado","Los valores por defecto se usan cuando los parametros son None","La funcion retorna un string no vacio","El evento activate limpia caches antiguos","La estrategia cache-first guarda respuestas de red en cache"]

notes:
  Funcion pura: no hace I/O, no tiene efectos secundarios, retorna siempre el mismo string dado los mismos inputs.

  El codigo JS generado usa `self.skipWaiting()` en install y `self.clients.claim()` en activate para activacion inmediata del SW sin necesidad de refrescar la pagina.

  La estrategia de fetch:
  - **Network-first** (para `/api/` por defecto): intenta la red, guarda la respuesta exitosa en cache, y usa el cache como fallback si la red falla. Garantiza datos frescos cuando hay conexion.
  - **Cache-first** (para todo lo demas): sirve desde cache si existe, si no va a red y guarda en cache. Optimo para assets estaticos (JS, CSS, imagenes) que cambian con el nombre de version.

  Para invalidar el cache al desplegar una nueva version, basta con cambiar `cache_name` (ej: `app-cache-v2`). El evento activate limpia automaticamente cualquier cache con nombre distinto al actual.

code:
  """Genera el codigo JavaScript de un Service Worker basico para PWAs."""

  from __future__ import annotations


  def generate_service_worker(
      cache_name: str = "app-cache-v1",
      precache_urls: list[str] | None = None,
      network_first_patterns: list[str] | None = None,
  ) -> str:
      """Genera el codigo JavaScript de un Service Worker con estrategia cache-first
      para assets estaticos y network-first para API calls.

      Args:
          cache_name: Nombre del cache a usar para esta version del SW.
          precache_urls: URLs a pre-cachear en el evento install.
          network_first_patterns: Patrones de URL que usan estrategia network-first.

      Returns:
          String con codigo JavaScript valido del Service Worker.
      """
      if precache_urls is None:
          precache_urls = ["/", "/index.html"]
      if network_first_patterns is None:
          network_first_patterns = ["/api/"]

      precache_urls_js = _list_to_js_array(precache_urls, indent=2)
      network_patterns_js = _list_to_js_array(network_first_patterns, indent=2)
      network_check_js = _build_network_check(network_first_patterns)

      return f"""\
  // Service Worker — generado automaticamente
  // Estrategia: cache-first para assets, network-first para API

  const CACHE_NAME = '{cache_name}';

  const PRECACHE_URLS = {precache_urls_js};

  const NETWORK_FIRST_PATTERNS = {network_patterns_js};

  // ── install ──────────────────────────────────────────────────────────────────
  // Pre-cachea los assets estaticos esenciales al instalar el SW.
  self.addEventListener('install', (event) => {{
    event.waitUntil(
      caches.open(CACHE_NAME).then((cache) => {{
        return cache.addAll(PRECACHE_URLS);
      }})
    );
    self.skipWaiting();
  }});

  // ── activate ─────────────────────────────────────────────────────────────────
  // Limpia caches de versiones anteriores al activar la nueva version del SW.
  self.addEventListener('activate', (event) => {{
    event.waitUntil(
      caches.keys().then((cacheNames) => {{
        return Promise.all(
          cacheNames
            .filter((name) => name !== CACHE_NAME)
            .map((name) => caches.delete(name))
        );
      }})
    );
    self.clients.claim();
  }});

  // ── fetch ─────────────────────────────────────────────────────────────────────
  // Intercepta peticiones:
  //   - URLs que coinciden con NETWORK_FIRST_PATTERNS → network-first con fallback a cache
  //   - El resto → cache-first con fallback a network (y guarda en cache la respuesta)
  self.addEventListener('fetch', (event) => {{
    const url = event.request.url;

    const isNetworkFirst = {network_check_js};

    if (isNetworkFirst) {{
      // Network-first: intenta red, si falla usa cache
      event.respondWith(
        fetch(event.request)
          .then((response) => {{
            if (response && response.status === 200) {{
              const responseClone = response.clone();
              caches.open(CACHE_NAME).then((cache) => {{
                cache.put(event.request, responseClone);
              }});
            }}
            return response;
          }})
          .catch(() => caches.match(event.request))
      );
    }} else {{
      // Cache-first: sirve desde cache, si no existe va a red y guarda en cache
      event.respondWith(
        caches.match(event.request).then((cached) => {{
          if (cached) {{
            return cached;
          }}
          return fetch(event.request).then((response) => {{
            if (response && response.status === 200) {{
              const responseClone = response.clone();
              caches.open(CACHE_NAME).then((cache) => {{
                cache.put(event.request, responseClone);
              }});
            }}
            return response;
          }});
        }})
      );
    }}
  }});
  """


  def _list_to_js_array(items: list[str], indent: int = 0) -> str:
      """Convierte una lista Python a un array JS multilínea."""
      if not items:
          return "[]"
      pad = " " * indent
      inner = f",\n{pad}  ".join(f"'{item}'" for item in items)
      return f"[\n{pad}  {inner},\n{pad}]"


  def _build_network_check(patterns: list[str]) -> str:
      """Genera la expresion JS que comprueba si una URL matchea algun patron."""
      if not patterns:
          return "false"
      checks = " || ".join(f"url.includes('{p}')" for p in patterns)
      return checks

params_schema:
  {"params":[{"name":"cache_name","desc":"Nombre del cache para esta version del SW. Se usa como clave para identificar el cache activo y limpiar los anteriores en activate."},{"name":"precache_urls","desc":"Lista de URLs a pre-cachear durante el evento install. Por defecto: ['/', '/index.html']."},{"name":"network_first_patterns","desc":"Lista de patrones de URL (substrings) que usan estrategia network-first. Por defecto: ['/api/']. Cualquier URL que contenga alguno de estos patrones va a red primero."}],"output":"String con codigo JavaScript valido del Service Worker listo para escribir como sw.js. Incluye handlers de install (precacheo), activate (limpieza de caches antiguos) y fetch (cache-first / network-first segun patron)."}


id: get_leaf_nodes_py_core
name: get_leaf_nodes
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def get_leaf_nodes(structure: Any) -> list[dict]
description: Extrae solo nodos hoja (sin hijos) de un arbol jerarquico. Deep copy de cada nodo.
tags: ["tree","leaf","hierarchy","functional"]
uses_functions: []
uses_types: []
returns: []
imports: ["copy"]
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Usa deep copy. Un nodo es hoja si su campo 'nodes' es falsy (vacio o ausente).
content_hash: 8c1103eeb5ce3a33e73ff94fc94636d9b2f41a91824b781e4ee81122131bfbfc
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  tree = [{"title": "A", "nodes": [{"title": "A1", "nodes": []}, {"title": "A2", "nodes": []}]}]
  get_leaf_nodes(tree)
  # [{"title": "A1"}, {"title": "A2"}]
  ```

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"structure","desc":"arbol jerarquico (dict con campo 'nodes' anidado recursivamente)"}],"output":"lista de dicts representando solo los nodos hoja (sin hijos)"}


id: get_pdf_page_tokens_py_core
name: get_pdf_page_tokens
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
description: Extrae texto y cuenta tokens por pagina de un PDF. Soporta PyPDF2 y PyMuPDF como backends.
tags: ["pdf","tokens","extraction","litellm","parsing"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["litellm","PyPDF2"]
tests: []
file_path: python/functions/core/get_pdf_page_tokens.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 69dfabef70ac2301b193f7307e3e438ea969a78dcbe5681e30ee1df958daeb97
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
signature:
  def get_pdf_page_tokens(pdf_path, model: str = None, pdf_parser: str = 'PyPDF2') -> list[tuple[str, int]]

example:
  ```python
  pages = get_pdf_page_tokens("report.pdf", model="gpt-4o")
  for text, tokens in pages:
      print(f"{tokens} tokens")

  # Con PyMuPDF (mejor para PDFs complejos)
  pages = get_pdf_page_tokens("report.pdf", pdf_parser="PyMuPDF")
  total = sum(t for _, t in pages)
  ```

notes:
  Requiere `pip install litellm PyPDF2` (o `pymupdf` para backend PyMuPDF). Acepta path string o BytesIO. Util para estimar costos de procesamiento LLM y para page_list_to_groups.

code:
  """Extract text and token count per page from a PDF. Supports PyPDF2 and PyMuPDF."""

  import os
  from io import BytesIO

  import litellm


  def get_pdf_page_tokens(pdf_path, model: str = None,
                          pdf_parser: str = "PyPDF2") -> list[tuple[str, int]]:
      """Extract text and token count for each page of a PDF.

      Args:
          pdf_path: Path to PDF file, or BytesIO object.
          model: Model name for token counting (passed to litellm.token_counter).
          pdf_parser: Parser backend — 'PyPDF2' or 'PyMuPDF'.

      Returns:
          list[tuple[str, int]]: List of (page_text, token_count) per page.
      """
      if pdf_parser == "PyPDF2":
          import PyPDF2
          pdf_reader = PyPDF2.PdfReader(pdf_path)
          page_list = []
          for page in pdf_reader.pages:
              page_text = page.extract_text() or ""
              token_length = litellm.token_counter(model=model, text=page_text)
              page_list.append((page_text, token_length))
          return page_list

      elif pdf_parser == "PyMuPDF":
          import pymupdf
          if isinstance(pdf_path, BytesIO):
              doc = pymupdf.open(stream=pdf_path, filetype="pdf")
          elif isinstance(pdf_path, str) and os.path.isfile(pdf_path):
              doc = pymupdf.open(pdf_path)
          else:
              raise ValueError(f"Invalid pdf_path: {pdf_path}")
          page_list = []
          for page in doc:
              page_text = page.get_text()
              token_length = litellm.token_counter(model=model, text=page_text)
              page_list.append((page_text, token_length))
          return page_list

      else:
          raise ValueError(f"Unsupported PDF parser: {pdf_parser}. Use 'PyPDF2' or 'PyMuPDF'.")

params_schema:
  {"params":[{"name":"pdf_path","desc":"ruta al archivo PDF (string o BytesIO)"},{"name":"model","desc":"nombre del modelo LLM para contar tokens (p.ej. 'gpt-4o')"},{"name":"pdf_parser","desc":"backend para extraer texto: 'PyPDF2' (default) o 'PyMuPDF'"}],"output":"lista de tuplas (text, token_count) uno por pagina del PDF"}


id: get_text_stats_py_core
name: get_text_stats
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def get_text_stats(text: str) -> dict
description: Estadisticas basicas de un texto: total de caracteres, lineas y palabras.
tags: ["text","statistics","stats","characters","words","lines"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: ["texto normal con palabras y lineas","texto vacio retorna ceros","texto con solo newlines"]
test_file_path: python/functions/core/get_text_stats_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 5d66780d6ed3368842288d6ba6567037a6a9dea9d6d341af873454bc784186cb
example:
  ```python
  stats = get_text_stats("hello world\nfoo bar")
  # {"total_chars": 19, "total_lines": 2, "total_words": 4}
  ```

notes:
  Funcion pura sin dependencias externas. `total_lines` cuenta newlines + 1, por lo que un texto vacio cuenta como 1 linea (comportamiento consistente con `wc -l` + 1). `total_words` usa `str.split()` que separa por cualquier whitespace y descarta vacios, equivalente a contar tokens separados por espacios.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"text","desc":"string de texto a analizar"}],"output":"dict con claves total_chars, total_lines, total_words conteniendo conteos basicos"}


id: group_by_py_core
name: group_by
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def group_by(xs: list, key_fn: callable) -> dict
description: Agrupa elementos de una lista por una funcion clave. Retorna dict de clave a lista.
tags: ["list","functional","group","classify","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. El orden de los elementos dentro de cada grupo se preserva.
content_hash: 95fbf958d07cf5ce351d71bce5ea0a80eb0834607e35a181cb8a8108edc5849c
example:
  ```python
  result = group_by(["hi", "hey", "bye"], lambda s: s[0])
  # {"h": ["hi", "hey"], "b": ["bye"]}
  ```

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"lista de elementos a agrupar"},{"name":"key_fn","desc":"funcion que extrae la clave de agrupacion de cada elemento"}],"output":"dict donde las keys son los resultados de key_fn y los values son listas de elementos"}


id: html_to_markdown_py_core
name: html_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: html_to_markdown(html: str) -> str
tags: ["html","markdown","parse","convert","readabilipy","markdownify","content-extraction"]
uses_functions: []
uses_types: []
returns: []
imports: ["re","html.parser"]
test_file_path: python/functions/core/html_to_markdown_test.py
file_path: python/functions/core/html_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 1d465f86cdb511528b2d27b785a01f6f5643e2365a2f53f0e80a4765b67e9ed3
description:
  Convierte HTML a markdown. Usa readabilipy para extraer contenido principal (filtra nav, ads, boilerplate), luego markdownify para convertir a markdown. Si las librerias opcionales no estan disponibles, usa un parser stdlib como fallback.

example:
  ```python
  from core.html_to_markdown import html_to_markdown

  html = """
  <html>
    <body>
      <nav><a href="/">Home</a></nav>
      <main>
        <h1>Titulo del articulo</h1>
        <p>Contenido <strong>relevante</strong> aqui.</p>
      </main>
      <footer>Copyright 2026</footer>
    </body>
  </html>
  """

  md = html_to_markdown(html)
  # "# Titulo del articulo\n\nContenido **relevante** aqui."
  ```

tests:
  ["HTML con nav/footer filtra boilerplate","HTML limpio se convierte correctamente","HTML con imagenes lazy-loaded"]

notes:
  Algoritmo:
  1. Preprocesar HTML: manejar contenido oculto WeChat (`js_content` con display:none),
     lazy loading images (`data-src` → `src`).
  2. Extraer contenido principal con `readabilipy` (basado en Mozilla Readability).
     Si no esta disponible, usa el HTML completo.
  3. Convertir a markdown con `markdownify` (headings ATX, strip script/style).
     Si no esta disponible, usa el parser stdlib de la misma funcion.

  Dependencias opcionales (mejoran la calidad si estan instaladas):
  - `readabilipy` — extraccion del contenido principal (filtra nav, ads, boilerplate)
  - `markdownify` — conversion HTML→markdown de alta fidelidad
  - `beautifulsoup4` — requerida por readabilipy

  Sin las dependencias opcionales la funcion sigue siendo pura y funcional,
  usando `html.parser` de stdlib como fallback.

  Funcion pura. No hace I/O ni tiene efectos secundarios.

code:
  """Convierte HTML a markdown usando readabilipy + markdownify, con fallback a stdlib."""

  import re
  from html.parser import HTMLParser
  from typing import Optional


  # ---------------------------------------------------------------------------
  # Stdlib fallback parser (no external deps)
  # ---------------------------------------------------------------------------

  _BLOCK_TAGS = {
      "p", "div", "article", "section", "main", "header", "footer", "aside",
      "nav", "figure", "figcaption", "blockquote", "pre", "ul", "ol", "li",
      "table", "thead", "tbody", "tr", "th", "td", "h1", "h2", "h3",
      "h4", "h5", "h6", "br", "hr",
  }

  _SKIP_TAGS = {
      "script", "style", "noscript", "iframe", "svg", "canvas",
      "nav", "footer", "header", "aside",
  }

  _HEADING_TAGS = {"h1": 1, "h2": 2, "h3": 3, "h4": 4, "h5": 5, "h6": 6}


  class _HTMLToMarkdownParser(HTMLParser):
      """Minimal HTML → Markdown parser using only stdlib."""

      def __init__(self) -> None:
          super().__init__(convert_charrefs=True)
          self._parts: list[str] = []
          self._skip_depth = 0
          self._in_pre = False
          self._tag_stack: list[str] = []
          self._list_stack: list[str] = []

      def handle_starttag(self, tag: str, attrs: list) -> None:
          tag = tag.lower()
          self._tag_stack.append(tag)

          if self._skip_depth > 0:
              if tag in _SKIP_TAGS:
                  self._skip_depth += 1
              return

          if tag in _SKIP_TAGS:
              self._skip_depth += 1
              return

          attrs_dict = dict(attrs)

          if tag in _HEADING_TAGS:
              level = _HEADING_TAGS[tag]
              self._parts.append(f"\n\n{'#' * level} ")

          elif tag == "p":
              self._parts.append("\n\n")

          elif tag == "br":
              self._parts.append("  \n")

          elif tag == "hr":
              self._parts.append("\n\n---\n\n")

          elif tag == "pre":
              self._in_pre = True
              self._parts.append("\n\n```\n")

          elif tag == "code" and not self._in_pre:
              self._parts.append("`")

          elif tag in ("strong", "b"):
              self._parts.append("**")

          elif tag in ("em", "i"):
              self._parts.append("*")

          elif tag == "a":
              href = attrs_dict.get("href", "")
              self._parts.append("[")
              self._parts.append(f"_href:{href}_")

          elif tag == "img":
              # Handle lazy-loaded images: prefer data-src over src
              src = attrs_dict.get("data-src") or attrs_dict.get("src", "")
              alt = attrs_dict.get("alt", "")
              self._parts.append(f"\n\n![{alt}]({src})\n\n")

          elif tag == "ul":
              self._list_stack.append("ul")
              self._parts.append("\n")

          elif tag == "ol":
              self._list_stack.append("ol")
              self._parts.append("\n")

          elif tag == "li":
              prefix = "-" if (not self._list_stack or self._list_stack[-1] == "ul") else "1."
              self._parts.append(f"\n{prefix} ")

          elif tag in ("blockquote",):
              self._parts.append("\n\n> ")

          elif tag in ("th", "td"):
              self._parts.append("| ")

          elif tag == "tr":
              self._parts.append("\n")

      def handle_endtag(self, tag: str) -> None:
          tag = tag.lower()
          if self._tag_stack and self._tag_stack[-1] == tag:
              self._tag_stack.pop()

          if self._skip_depth > 0:
              if tag in _SKIP_TAGS:
                  self._skip_depth -= 1
              return

          if tag in _HEADING_TAGS:
              self._parts.append("\n\n")

          elif tag == "p":
              self._parts.append("\n\n")

          elif tag == "pre":
              self._in_pre = False
              self._parts.append("\n```\n\n")

          elif tag == "code" and not self._in_pre:
              self._parts.append("`")

          elif tag in ("strong", "b"):
              self._parts.append("**")

          elif tag in ("em", "i"):
              self._parts.append("*")

          elif tag == "a":
              # Find the matching _href: placeholder and rebuild [text](href)
              text_parts: list[str] = []
              href = ""
              while self._parts:
                  part = self._parts.pop()
                  if part.startswith("_href:") and part.endswith("_"):
                      href = part[6:-1]
                      # collected text_parts in reverse, also the "[" opener
                      if self._parts and self._parts[-1] == "[":
                          self._parts.pop()
                      break
                  text_parts.insert(0, part)
              link_text = "".join(text_parts).strip()
              self._parts.append(f"[{link_text}]({href})")

          elif tag in ("ul", "ol"):
              if self._list_stack:
                  self._list_stack.pop()
              self._parts.append("\n")

      def handle_data(self, data: str) -> None:
          if self._skip_depth > 0:
              return
          if self._in_pre:
              self._parts.append(data)
          else:
              self._parts.append(data)

      def get_markdown(self) -> str:
          raw = "".join(self._parts)
          # Collapse 3+ consecutive newlines to 2
          raw = re.sub(r"\n{3,}", "\n\n", raw)
          return raw.strip()


  def _stdlib_html_to_markdown(html: str) -> str:
      """Convert HTML to markdown using only Python stdlib."""
      parser = _HTMLToMarkdownParser()
      parser.feed(html)
      return parser.get_markdown()


  # ---------------------------------------------------------------------------
  # Public function
  # ---------------------------------------------------------------------------


  def html_to_markdown(html: str) -> str:
      """Convierte HTML a markdown.

      Usa readabilipy para extraer el contenido principal (filtra nav, ads,
      boilerplate) y markdownify para convertir a markdown. Si alguna de esas
      librerias no esta disponible, usa un parser stdlib como fallback.

      Pasos:
      1. Preprocesar HTML: manejar contenido oculto (WeChat js_content),
         lazy loading images (data-src → src).
      2. Extraer contenido principal con readabilipy (basado en Mozilla
         Readability). Fallback: usar el HTML completo.
      3. Convertir a markdown con markdownify (headings ATX, strip
         script/style). Fallback: parser stdlib.

      Args:
          html: HTML completo de la pagina.

      Returns:
          Contenido de la pagina en formato markdown.
      """
      # Step 1: preprocess — handle WeChat hidden content and lazy-loaded images
      html = _preprocess_html(html)

      # Step 2: extract main content with readabilipy (optional dep)
      main_html = _extract_main_content(html)

      # Step 3: convert to markdown
      return _convert_to_markdown(main_html)


  def _preprocess_html(html: str) -> str:
      """Preprocesar HTML antes de extraer contenido.

      - Expande contenido oculto de WeChat (js_content).
      - Reemplaza data-src por src en imagenes lazy-loaded.
      """
      # WeChat js_content: replace hidden wrapper divs
      html = re.sub(
          r'<div[^>]*id=["\']js_content["\'][^>]*style=["\'][^"\']*display\s*:\s*none[^"\']*["\'][^>]*>',
          '<div id="js_content">',
          html,
          flags=re.IGNORECASE,
      )

      # Lazy loading: copy data-src to src for img tags
      def replace_lazy_src(m: re.Match) -> str:
          tag = m.group(0)
          data_src_match = re.search(r'data-src=["\']([^"\']*)["\']', tag)
          if data_src_match:
              data_src = data_src_match.group(1)
              # Replace or add src attribute
              if re.search(r'\bsrc=["\']', tag):
                  tag = re.sub(r'\bsrc=["\'][^"\']*["\']', f'src="{data_src}"', tag)
              else:
                  tag = tag.replace("<img", f'<img src="{data_src}"', 1)
          return tag

      html = re.sub(r"<img[^>]+>", replace_lazy_src, html, flags=re.IGNORECASE)
      return html


  def _extract_main_content(html: str) -> str:
      """Extraer contenido principal usando readabilipy si esta disponible."""
      try:
          from readabilipy import simple_json_from_html_string  # type: ignore

          article = simple_json_from_html_string(html, use_readability=True)
          return article.get("content") or html
      except ImportError:
          return html


  def _convert_to_markdown(html: str) -> str:
      """Convertir HTML a markdown usando markdownify si esta disponible."""
      try:
          import markdownify  # type: ignore

          return markdownify.markdownify(
              html,
              heading_style="ATX",
              strip=["script", "style"],
          )
      except ImportError:
          return _stdlib_html_to_markdown(html)

params_schema:
  {"params":[{"name":"html","desc":"string con contenido HTML a convertir"}],"output":"string con markdown equivalente al contenido HTML"}


id: is_git_repo_url_py_core
name: is_git_repo_url
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def is_git_repo_url(url: str, known_hosts: list[str] | None = None) -> bool
tags: ["git","url","validation","github","gitlab","repository"]
uses_functions: []
uses_types: []
returns: []
imports: ["urllib.parse"]
tests: ["URL repo valida","URL de issue (False)","URL de blob/file (False)","URL con tree/branch (True)"]
test_file_path: python/functions/core/parse_git_url_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 5aab8611784b855c1a96fffc93455ac697f4feb9206889d1ec32535a05b8fbce
description:
  Verifica si una URL apunta a un repositorio git clonable. Acepta org/repo y org/repo/tree/<ref>. Rechaza issues, blobs, PRs y otros sub-recursos.

example:
  ```python
  is_git_repo_url("https://github.com/psf/requests")
  # True

  is_git_repo_url("https://github.com/psf/requests/issues/123")
  # False

  is_git_repo_url("https://github.com/psf/requests/blob/main/README.md")
  # False

  is_git_repo_url("https://github.com/psf/requests/tree/main")
  # True

  is_git_repo_url("git@github.com:psf/requests.git")
  # True
  ```

notes:
  Funcion pura. Para SSH y git:// se acepta cualquier path siempre que el host sea conocido (los protocolos de clonacion no navegan a sub-recursos). Para HTTP/HTTPS se exige exactamente 2 segmentos (org/repo) o 4 segmentos con `tree` en posicion 3.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"url","desc":"URL del repositorio de código a validar (HTTPS, HTTP, SSH)"},{"name":"known_hosts","desc":"Lista opcional de hosts conocidos para validar URLs SSH"}],"output":"True si la URL apunta a un repositorio clonable, False en caso contrario"}


id: join_by_key_py_core
name: join_by_key
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def join_by_key(left: list[dict], right: list[dict], key: str, how: str = 'inner') -> list[dict]
tags: ["tabular","join","merge","python","core"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/join_by_key_test.py
file_path: python/functions/core/join_by_key.py
created_at: 2026-04-05T11:24:52Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 038c35f71009d9f059a816900b4a6b073a44bb08df2c8561e804c66ed4d6ac05
description:
  Join de dos listas de dicts por una clave comun. Soporta inner, left, right y outer. Campos duplicados del right se sufijan con _right. Algoritmo O(n+m).

example:
  ```python
  left = [{"id": 1, "name": "Alice"}, {"id": 2, "name": "Bob"}]
  right = [{"id": 1, "dept": "eng"}, {"id": 3, "dept": "sales"}]

  join_by_key(left, right, key="id", how="inner")
  # [{"id": 1, "name": "Alice", "dept": "eng"}]

  join_by_key(left, right, key="id", how="left")
  # [{"id": 1, "name": "Alice", "dept": "eng"},
  #  {"id": 2, "name": "Bob", "dept": None}]
  ```

tests:
  ["Inner join solo matches","Left join todos los left con None para right sin match","Right join","Outer join","Campos duplicados con sufijo _right","Key ausente en alguna fila"]

notes:
  Funcion pura sin dependencias externas.
  El algoritmo indexa right en O(n) y luego itera left en O(m), total O(n+m).
  Los campos de right que colisionan con campos de left (excepto la clave) se renombran con sufijo _right.

code:
  """Join de dos tablas tabulares por una clave comun."""


  def join_by_key(
      left: list[dict],
      right: list[dict],
      key: str,
      how: str = "inner",
  ) -> list[dict]:
      """Une dos listas de dicts por una clave comun.

      Soporta los cuatro tipos de join: inner, left, right, outer.
      Campos duplicados del lado right (distintos a la clave) se sufijan con _right.

      Algoritmo O(n+m): indexa right por key, luego itera left buscando matches.

      Args:
          left: Lista de dicts del lado izquierdo.
          right: Lista de dicts del lado derecho.
          key: Nombre del campo clave para el join.
          how: Tipo de join: inner, left, right, outer.

      Returns:
          Lista de dicts con campos de ambos lados mergeados.
          Campos del right ausentes en un match left se rellenan con None.
          Campos del left ausentes en un match right se rellenan con None.
      """
      # Indexar right por key
      right_index: dict[any, list[dict]] = {}
      for row in right:
          k = row.get(key)
          right_index.setdefault(k, []).append(row)

      # Determinar campos del right que podrian colisionar con left
      left_keys = {k for row in left for k in row}
      right_only_keys = {k for row in right for k in row if k != key}
      conflicting = right_only_keys & left_keys - {key}

      def _merge(l_row: dict | None, r_row: dict | None) -> dict:
          result: dict = {}
          if l_row is not None:
              result.update(l_row)
          if r_row is not None:
              for k, v in r_row.items():
                  if k == key:
                      continue
                  if k in conflicting:
                      result[f"{k}_right"] = v
                  else:
                      result[k] = v
          return result

      def _empty_left(left_sample: dict | None) -> dict:
          if left_sample is None:
              return {}
          return {k: None for k in left_sample}

      def _empty_right() -> dict:
          result: dict = {}
          for row in right:
              for k in row:
                  if k == key:
                      continue
                  dest = f"{k}_right" if k in conflicting else k
                  result[dest] = None
          return result

      matched_right_keys: set = set()
      output: list[dict] = []

      for l_row in left:
          k = l_row.get(key)
          r_rows = right_index.get(k)
          if r_rows:
              matched_right_keys.add(k)
              for r_row in r_rows:
                  output.append(_merge(l_row, r_row))
          else:
              if how in ("left", "outer"):
                  output.append(_merge(l_row, None) | _empty_right())

      if how in ("right", "outer"):
          for r_row in right:
              k = r_row.get(key)
              if k not in matched_right_keys:
                  base = _empty_right()
                  base[key] = k
                  for rk, rv in r_row.items():
                      if rk == key:
                          continue
                      dest = f"{rk}_right" if rk in conflicting else rk
                      base[dest] = rv
                  output.append(base)

      return output

params_schema:
  {"params":[{"name":"left","desc":"Lista izquierda de diccionarios a joinear"},{"name":"right","desc":"Lista derecha de diccionarios a joinear"},{"name":"key","desc":"Nombre de la clave común para hacer matching"},{"name":"how","desc":"Tipo de join: 'inner', 'left', 'right' u 'outer'"}],"output":"Lista de diccionarios resultantes del join con campos renombrados si hay conflicto"}


id: list_to_tree_py_core
name: list_to_tree
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def list_to_tree(data: list[dict]) -> list[dict]
description: Convierte lista plana con codigos de estructura ('1.2.3') a arbol jerarquico anidado.
tags: ["tree","hierarchy","structure","conversion"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 958d2f62de9dba3b954400a8f6c55f21d40c922586182ff684fd70b062d1d1bc
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
example:
  ```python
  flat = [
      {"structure": "1", "title": "Intro", "start_index": 1, "end_index": 5},
      {"structure": "1.1", "title": "Background", "start_index": 1, "end_index": 3},
      {"structure": "1.2", "title": "Scope", "start_index": 3, "end_index": 5},
      {"structure": "2", "title": "Methods", "start_index": 5, "end_index": 10},
  ]
  tree = list_to_tree(flat)
  # [{"title": "Intro", "nodes": [{"title": "Background"}, {"title": "Scope"}]}, {"title": "Methods"}]
  ```

notes:
  Funcion pura. Cada item necesita campo 'structure' con codigo jerarquico separado por puntos. Nodos huerfanos se promueven a raiz.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"data","desc":"Lista plana de diccionarios con campo 'structure' (ej: '1.2.3')"}],"output":"Árbol jerárquico anidado donde nodos padres contienen su lista 'nodes'"}


id: llm_acompletion_retry_py_core
name: llm_acompletion_retry
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["llm","completion","retry","async","litellm","api"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["litellm","asyncio","logging"]
tests: []
file_path: python/functions/core/llm_acompletion_retry.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: c1919dd7d4bbafcb4ca0cf164dc65142b0d5e4bcccd4b7b2f1e597d4a76c1222
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
signature:
  async def llm_acompletion_retry(model: str, prompt: str, max_retries: int = 10, temperature: float = 0) -> str

description:
  Completion LLM asincrono con retry automatico. Soporte multi-modelo via litellm (OpenAI, Anthropic, etc.).

example:
  ```python
  import asyncio

  async def main():
      response = await llm_acompletion_retry("gpt-4o", "Summarize this text: ...")
      print(response)

  asyncio.run(main())
  ```

notes:
  Requiere `pip install litellm`. Version async de llm_completion_retry. Usa asyncio.sleep entre retries. Ideal para procesar multiples prompts en paralelo con asyncio.gather.

code:
  """Async LLM completion with retry logic via litellm. Supports 100+ models."""

  import asyncio
  import logging

  import litellm

  litellm.drop_params = True


  async def llm_acompletion_retry(model: str, prompt: str, max_retries: int = 10,
                                  temperature: float = 0) -> str:
      """Asynchronous LLM completion with retry. Multi-model support via litellm.

      Args:
          model: Model name (e.g. 'gpt-4o', 'claude-sonnet-4-20250514'). Strips 'litellm/' prefix.
          prompt: User prompt text.
          max_retries: Max retry attempts on failure.
          temperature: Sampling temperature.

      Returns:
          str: Response content. Empty string if all retries fail.
      """
      if model:
          model = model.removeprefix("litellm/")

      messages = [{"role": "user", "content": prompt}]

      for i in range(max_retries):
          try:
              response = await litellm.acompletion(
                  model=model,
                  messages=messages,
                  temperature=temperature,
              )
              return response.choices[0].message.content
          except Exception as e:
              logging.error(f"Async LLM completion error (attempt {i+1}/{max_retries}): {e}")
              if i < max_retries - 1:
                  await asyncio.sleep(1)
              else:
                  logging.error(f"Max retries reached for model={model}")
                  return ""

params_schema:
  {"params":[{"name":"model","desc":"Nombre del modelo LLM (ej: 'gpt-4o', 'claude-sonnet-4-20250514')"},{"name":"prompt","desc":"Mensaje de usuario a enviar al modelo"},{"name":"max_retries","desc":"Número máximo de reintentos ante errores transitorios"},{"name":"temperature","desc":"Temperatura para la generación (0.0 a 1.0)"}],"output":"Texto de respuesta del modelo LLM"}


id: llm_completion_retry_py_core
name: llm_completion_retry
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["llm","completion","retry","litellm","api"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["litellm","logging","time"]
tests: []
file_path: python/functions/core/llm_completion_retry.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 93da6872007658e6f8fc13bbd5e158a71bff0c2b25323721995a680ac19d9577
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/utils.py
signature:
  def llm_completion_retry(model: str, prompt: str, chat_history: list = None, return_finish_reason: bool = False, max_retries: int = 10, temperature: float = 0) -> str

description:
  Completion LLM sincrono con retry automatico (max 10). Soporte multi-modelo via litellm (OpenAI, Anthropic, etc.).

example:
  ```python
  response = llm_completion_retry("gpt-4o", "Explain quantum computing in one sentence")
  # "Quantum computing uses quantum bits..."

  # Con historial de chat
  history = [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello!"}]
  response = llm_completion_retry("claude-sonnet-4-20250514", "What's 2+2?", chat_history=history)

  # Con finish reason
  content, reason = llm_completion_retry("gpt-4o", "...", return_finish_reason=True)
  # reason: "finished" | "max_output_reached" | "error"
  ```

notes:
  Requiere `pip install litellm`. Soporta 100+ modelos via litellm. Retry con sleep(1) entre intentos. Retorna string vacio si todos los intentos fallan.

code:
  """LLM completion with retry logic via litellm. Supports 100+ models."""

  import logging
  import time

  import litellm

  litellm.drop_params = True


  def llm_completion_retry(model: str, prompt: str, chat_history: list = None,
                           return_finish_reason: bool = False, max_retries: int = 10,
                           temperature: float = 0):
      """Synchronous LLM completion with retry. Multi-model support via litellm.

      Args:
          model: Model name (e.g. 'gpt-4o', 'claude-sonnet-4-20250514'). Strips 'litellm/' prefix.
          prompt: User prompt text.
          chat_history: Optional list of prior messages [{"role": ..., "content": ...}].
          return_finish_reason: If True, returns (content, reason) tuple.
          max_retries: Max retry attempts on failure.
          temperature: Sampling temperature.

      Returns:
          str or (str, str): Response content, optionally with finish reason.
      """
      if model:
          model = model.removeprefix("litellm/")

      messages = list(chat_history or []) + [{"role": "user", "content": prompt}]

      for i in range(max_retries):
          try:
              response = litellm.completion(
                  model=model,
                  messages=messages,
                  temperature=temperature,
              )
              content = response.choices[0].message.content
              if return_finish_reason:
                  reason = "max_output_reached" if response.choices[0].finish_reason == "length" else "finished"
                  return content, reason
              return content
          except Exception as e:
              logging.error(f"LLM completion error (attempt {i+1}/{max_retries}): {e}")
              if i < max_retries - 1:
                  time.sleep(1)
              else:
                  logging.error(f"Max retries reached for model={model}")
                  if return_finish_reason:
                      return "", "error"
                  return ""

params_schema:
  {"params":[{"name":"model","desc":"Nombre del modelo LLM soportado por litellm"},{"name":"prompt","desc":"Prompt del usuario a procesar"},{"name":"chat_history","desc":"Historial de chat opcional con formato [{'role': '...', 'content': '...'}]"},{"name":"return_finish_reason","desc":"Si True, retorna tupla (content, finish_reason) en lugar de solo content"},{"name":"max_retries","desc":"Número máximo de reintentos ante fallos"},{"name":"temperature","desc":"Temperatura para la generación"}],"output":"String con la respuesta del modelo, o tupla (str, str) si return_finish_reason=True"}


id: load_translations_py_core
name: load_translations
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def load_translations(locales_dir: str) -> dict[str, dict]
tags: ["i18n","translation","locale","json","files"]
uses_functions: []
uses_types: []
returns: []
error_type: error_go_core
imports: ["json","os"]
test_file_path: python/functions/core/load_translations_test.py
file_path: python/functions/core/load_translations.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 843fb3c59250898043ddc58efe16b69705608e7648185632b159af01529581a8
description:
  Carga todos los archivos JSON de un directorio de locales. Cada archivo {locale}.json se indexa por nombre sin extension. Retorna {} si el directorio no existe o esta vacio.

example:
  ```python
  from load_translations import load_translations
  from t import _set_translations, t

  # Estructura de archivos:
  # locales/
  #   en.json  →  {"report": {"done": "Done", "sectionStart": "Section: {title}"}}
  #   es.json  →  {"report": {"done": "Listo"}}

  translations = load_translations("locales/")
  _set_translations(translations, default_locale="en")

  t("report.done", locale="es")
  # → "Listo"
  ```

tests:
  ["carga multiples locales","directorio inexistente retorna dict vacio","ignora archivos no json","locale con estructura anidada"]

notes:
  Lee el filesystem, por eso es impura. Los errores de JSON malformado se propagan directamente (`json.JSONDecodeError`). Los errores de acceso al directorio se propagan como `OSError`. Companera natural de `t_py_core` — el flujo tipico es: `load_translations` al inicio de la app → `_set_translations` → llamadas a `t` durante la ejecucion. Inspirada conceptualmente en el modulo `locale.py` de MiroFish (AGPL-3.0); reimplementada desde cero.

code:
  """Carga de archivos JSON de un directorio de locales."""

  import json
  import os


  def load_translations(locales_dir: str) -> dict[str, dict]:
      """Carga todos los archivos JSON de un directorio de locales.

      Cada archivo `{locale}.json` se carga como diccionario y se indexa
      por el nombre del archivo sin extension (el locale).

      Args:
          locales_dir: Ruta al directorio que contiene los archivos JSON de locales.

      Returns:
          Diccionario {locale: dict_de_traducciones}. Retorna {} si el directorio
          no existe o no contiene archivos JSON.

      Raises:
          OSError: Si el directorio no es accesible.
          json.JSONDecodeError: Si un archivo JSON esta malformado.

      Example:
          >>> # locales/en.json = {"greeting": "Hello"}
          >>> # locales/es.json = {"greeting": "Hola"}
          >>> translations = load_translations("locales/")
          >>> translations["en"]["greeting"]
          'Hello'
          >>> translations["es"]["greeting"]
          'Hola'
      """
      translations: dict[str, dict] = {}

      if not os.path.isdir(locales_dir):
          return translations

      for filename in os.listdir(locales_dir):
          if not filename.endswith(".json"):
              continue
          locale = filename[:-5]  # quitar ".json"
          filepath = os.path.join(locales_dir, filename)
          with open(filepath, encoding="utf-8") as f:
              translations[locale] = json.load(f)

      return translations

params_schema:
  {"params":[{"name":"locales_dir","desc":"Ruta al directorio con archivos de traducción JSON ({locale}.json)"}],"output":"Diccionario con clave=nombre_locale y valor=contenido del JSON"}


id: map_list_py_core
name: map_list
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def map_list(xs: list, fn: callable) -> list
description: Aplica una funcion a cada elemento de una lista, retornando una nueva lista.
tags: ["list","functional","map","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
example:
  ```python
  doubled = map_list([1, 2, 3], lambda n: n * 2)
  # [2, 4, 6]
  ```

tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. No muta la lista original. Equivalente a `[fn(x) for x in xs]`.
content_hash: a3c5808241deefb5870a616b8676fcb6d17296e1c9b48531b0f80426d865af56
code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"Lista de elementos a transformar"},{"name":"fn","desc":"Función a aplicar a cada elemento"}],"output":"Nueva lista con el resultado de aplicar fn a cada elemento"}


id: match_pois_to_interests_py_core
name: match_pois_to_interests
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["poi","ranking","filtering","interests","geospatial","recommendation","pure"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/match_pois_to_interests_test.py
file_path: python/functions/core/match_pois_to_interests.py
created_at: 2026-04-06T16:23:12Z
updated_at: 2026-04-06T16:23:12Z
props: []
emits: []
variant: []
content_hash: 31b4d462aa010a5acde0f2e18683933b27346d32cadb090a629571a1448aa976
signature:
  def match_pois_to_interests(pois: list[dict], interests: list[dict], max_results: int = 10) -> list[dict]

description:
  Filtra y rankea una lista de POIs segun un perfil de intereses del usuario. Calcula un score por categoria (0.5), nombre (0.3) y tags (0.2), multiplicado por el weight del interes. Retorna los top max_results POIs con score > 0, enriquecidos con score y matched_interests.

example:
  ```python
  pois = [
      {"id": 1, "lat": 40.41, "lon": -3.70, "name": "Museo del Prado", "category": "museum", "tags": {"tourism": "museum"}},
      {"id": 2, "lat": 40.42, "lon": -3.71, "name": "Bar El Tigre", "category": "bar", "tags": {"amenity": "bar"}},
  ]
  interests = [{"name": "arte", "keywords": ["museo", "museum", "arte"], "weight": 1.0}]
  result = match_pois_to_interests(pois, interests)
  # [{"id": 1, ..., "score": 1.0, "matched_interests": ["arte"]}]
  ```

tests:
  ["pois que matchean parcialmente por categoria y nombre","pois sin match retorna lista vacia","multiples intereses con diferentes weights acumulan score"]

notes:
  Funcion pura: no hace I/O, no muta los inputs, es determinista.

  **Calculo de score por interes:**
  - Match de categoria: 0.5 si `poi["category"]` esta en los keywords del interes
  - Match de nombre: 0.3 si algun keyword aparece como substring en `poi["name"]` (case-insensitive)
  - Match de tags: 0.2 si algun keyword aparece en los valores de `poi["tags"]` (case-insensitive)
  - El subtotal del interes se multiplica por `weight`

  Si un POI matchea multiples intereses, los scores se suman. Un POI puede obtener score maximo de `sum(weight_i * 1.0)` sobre todos los intereses que matcheen.

  El campo `tags` puede ser dict con valores string o numericos — todos se convierten a string para la comparacion.

code:
  """Filtra y rankea POIs segun un perfil de intereses del usuario."""


  def match_pois_to_interests(
      pois: list[dict],
      interests: list[dict],
      max_results: int = 10,
  ) -> list[dict]:
      """Filtra y rankea una lista de POIs segun un perfil de intereses.

      Calcula un score para cada POI basado en matches de categoria, nombre
      y tags contra los keywords de cada interes. Solo retorna POIs con score > 0,
      ordenados por score descendente, hasta max_results.

      Args:
          pois: Lista de POIs con campos id, lat, lon, name, category, tags.
          interests: Lista de intereses con campos name, keywords y weight.
          max_results: Numero maximo de resultados a retornar.

      Returns:
          Lista de POIs enriquecidos con score y matched_interests, ordenados
          por score descendente. Lista vacia si ningun POI matchea.
      """
      scored: list[dict] = []

      for poi in pois:
          total_score = 0.0
          matched: list[str] = []

          category = (poi.get("category") or "").lower()
          name = (poi.get("name") or "").lower()
          tags = poi.get("tags") or {}
          tag_values = " ".join(str(v) for v in tags.values()).lower()

          for interest in interests:
              keywords: list[str] = [kw.lower() for kw in (interest.get("keywords") or [])]
              weight: float = float(interest.get("weight", 1.0))
              interest_name: str = interest.get("name", "")

              interest_score = 0.0

              # Match de categoria: 0.5 puntos si la categoria aparece en keywords
              if any(kw == category for kw in keywords):
                  interest_score += 0.5

              # Match de nombre: 0.3 puntos si algun keyword es substring del nombre
              if any(kw in name for kw in keywords):
                  interest_score += 0.3

              # Match de tags: 0.2 puntos si algun keyword aparece en valores de tags
              if any(kw in tag_values for kw in keywords):
                  interest_score += 0.2

              if interest_score > 0:
                  total_score += interest_score * weight
                  matched.append(interest_name)

          if total_score > 0:
              scored.append({**poi, "score": round(total_score, 6), "matched_interests": matched})

      scored.sort(key=lambda p: p["score"], reverse=True)
      return scored[:max_results]

params_schema:
  {"params":[{"name":"pois","desc":"Lista de POIs, cada uno con campos id, lat, lon, name, category y tags (dict). Formato compatible con la salida de overpass_nearby_pois."},{"name":"interests","desc":"Lista de intereses del usuario, cada uno con name (str), keywords (list[str]) y weight (float entre 0 y 1). Los keywords se comparan contra categoria, nombre y tags del POI."},{"name":"max_results","desc":"Numero maximo de POIs a retornar. Por defecto 10."}],"output":"Lista de POIs filtrados y ordenados por score descendente. Cada POI incluye todos sus campos originales mas score (float) y matched_interests (list[str] con los nombres de intereses que matchearon). Lista vacia si ningun POI supera score 0."}


id: merge_entity_attributes_py_core
name: merge_entity_attributes
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def merge_entity_attributes(attr_list: list[dict]) -> dict
tags: ["merge","entity","attributes","resolution","deduplication","fuzzygraph","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/merge_entity_attributes_test.py
file_path: python/functions/core/merge_entity_attributes.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: c873027aea78de507dab0bf47af91e9e5704487041246d29fc973326c0ac9904
description:
  Combina atributos de multiples candidatos de la misma entidad. Aplica heuristicas de resolucion por tipo de campo: max para numericos, min/max para fechas, union para listas, OR para booleanos, mas largo para strings.

example:
  ```python
  a = {"risk_score": 3.5, "first_seen": "2022-05-15", "verified": False}
  b = {"risk_score": 7.2, "first_seen": "2023-01-01", "verified": True, "alias": "Alice"}

  result = merge_entity_attributes([a, b])
  # {
  #   "risk_score": 7.2,           # max
  #   "first_seen": "2022-05-15",  # min (mas antigua)
  #   "verified": True,            # OR logico
  #   "alias": "Alice"             # solo en b
  # }
  ```

tests:
  ["Atributos complementarios (A tiene full_name, B tiene nationality) -\u003e ambos","Atributos conflictivos en risk_score -\u003e max","Atributos first_seen conflictivos -\u003e min","Todos null -\u003e null","Listas -\u003e union sin duplicados","Boolean verified -\u003e True si alguno es True","String conflictivo -\u003e usar el mas largo","Valores iguales -\u003e usar ese valor","Un solo candidato -\u003e retorna sus atributos tal cual","Lista vacia -\u003e retorna dict vacio","last_seen conflictivo -\u003e max (mas reciente)","Un candidato tiene null, otro tiene valor -\u003e usar el valor"]

notes:
  Funcion pura. No tiene dependencias externas. Las listas conocidas de campos especiales (`_NUMERIC_FIELDS`, `_DATE_MIN_FIELDS`, etc.) pueden extenderse si el dominio crece.

  Disenada originalmente para el grafo de entidades de fuzzygraph, donde multiples fuentes pueden describir la misma entidad con datos complementarios o contradictorios.

documentation:
  ## Heuristicas de resolucion

  | Campo / tipo | Conflicto | Resolucion |
  |---|---|---|
  | `risk_score`, `balance`, `cvss` | numerico | `max` |
  | `first_seen`, `created_date` | fecha | `min` (mas antigua) |
  | `last_seen`, `expires_date` | fecha | `max` (mas reciente) |
  | `verified`, `exploited` | booleano | `any` (OR logico) |
  | cualquier `list` | lista | union sin duplicados |
  | cualquier `str` u otro | string | el mas largo |

  Los campos fuera de las listas conocidas usan la heuristica por tipo Python (`list`, `bool`, luego `str`/otro).

code:
  """Combina atributos de multiples candidatos de la misma entidad."""

  from __future__ import annotations

  _NUMERIC_FIELDS = {"risk_score", "balance", "cvss"}
  _DATE_MIN_FIELDS = {"first_seen", "created_date"}
  _DATE_MAX_FIELDS = {"last_seen", "expires_date"}
  _BOOL_FIELDS = {"verified", "exploited"}


  def merge_entity_attributes(attr_list: list[dict]) -> dict:
      """Combina atributos de multiples candidatos de la misma entidad.

      Para cada campo presente en cualquier candidato recopila todos los valores
      non-null y aplica heuristicas de resolucion por tipo de campo:
      - Numerico (risk_score, balance, cvss): max
      - Fecha min (first_seen, created_date): min (mas antigua)
      - Fecha max (last_seen, expires_date): max (mas reciente)
      - Lista (cualquier valor de tipo list): union sin duplicados
      - Boolean (verified, exploited): OR logico
      - String: el mas largo

      Args:
          attr_list: Lista de dicts con los atributos de cada candidato.

      Returns:
          Dict con los atributos fusionados.
      """
      if not attr_list:
          return {}

      # Recopilar todas las claves presentes en cualquier candidato
      all_keys: set[str] = set()
      for attrs in attr_list:
          all_keys.update(attrs.keys())

      merged: dict = {}

      for key in all_keys:
          # Recopilar valores non-null
          values = [attrs[key] for attrs in attr_list if key in attrs and attrs[key] is not None]

          if not values:
              merged[key] = None
              continue

          if len(values) == 1:
              merged[key] = values[0]
              continue

          # Todos iguales
          if all(v == values[0] for v in values):
              merged[key] = values[0]
              continue

          # Resolver conflicto segun tipo de campo
          if key in _NUMERIC_FIELDS:
              merged[key] = max(values)
          elif key in _DATE_MIN_FIELDS:
              merged[key] = min(values)
          elif key in _DATE_MAX_FIELDS:
              merged[key] = max(values)
          elif key in _BOOL_FIELDS:
              merged[key] = any(values)
          elif isinstance(values[0], list):
              # Union de listas sin duplicados, preservando orden de aparicion
              seen: list = []
              for lst in values:
                  for item in lst:
                      if item not in seen:
                          seen.append(item)
              merged[key] = seen
          else:
              # String u otro: usar el mas largo
              str_values = [str(v) for v in values]
              merged[key] = max(str_values, key=len)

      return merged

params_schema:
  {"params":[{"name":"attr_list","desc":"Lista de diccionarios con atributos de candidatos de la misma entidad"}],"output":"Diccionario con atributos merged aplicando heurísticas de resolución por tipo"}


id: next_cron_time_py_core
name: next_cron_time
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: next_cron_time(schedule: dict, after: datetime) -> datetime | None
tags: ["cron","scheduling","time","next","pure"]
uses_functions: ["parse_cron_expr_py_core"]
uses_types: []
returns: []
imports: ["datetime"]
test_file_path: python/functions/core/next_cron_time_test.py
file_path: python/functions/core/next_cron_time.py
created_at: 2026-04-05T14:44:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: cea085d6fa611c9a9e89b8ccb757608301afb19f1f4b46578a5958f1e3b83e6d
description:
  Calcula la proxima ejecucion de un cron schedule despues de un tiempo dado. Avanza minuto a minuto saltando campos no coincidentes. Retorna None si no hay match en 366 dias (schedule imposible).

example:
  ```python
  from parse_cron_expr import parse_cron_expr
  from next_cron_time import next_cron_time
  from datetime import datetime

  sched = parse_cron_expr("0 * * * *")
  after = datetime(2024, 1, 15, 14, 30)
  nxt = next_cron_time(sched, after)
  # nxt = datetime(2024, 1, 15, 15, 0)

  weekdays = parse_cron_expr("0 9 * * 1-5")
  friday = datetime(2024, 1, 19, 10, 0)  # viernes
  nxt2 = next_cron_time(weekdays, friday)
  # nxt2 = datetime(2024, 1, 22, 9, 0)  # lunes
  ```

tests:
  ["0 * * * * desde :30 retorna la proxima hora en punto","0 9 * * 1-5 desde viernes retorna proximo lunes a las 9","schedule imposible retorna None","paso */15 desde :05 retorna :15"]

notes:
  Misma semantica que la version Go (next_cron_time_go_core). Usa AND para day_of_month y day_of_week.
  El campo day_of_week usa convencion cron (0=domingo) convertido internamente a weekday() de Python.
  Retorna None en lugar de lanzar excepcion para mantener purity (sin side effects).

code:
  """Calcula la proxima ejecucion de un cron schedule."""

  from __future__ import annotations

  from datetime import datetime, timedelta


  def next_cron_time(schedule: dict, after: datetime) -> datetime | None:
      """Calcula la proxima ejecucion de un cron schedule despues de `after`.

      Avanza minuto a minuto saltando campos no coincidentes. Retorna None
      si no hay match en 366 dias (schedule imposible).

      Args:
          schedule: Dict con claves minute, hour, day_of_month, month, day_of_week,
                    cada una con lista de enteros validos. Producido por parse_cron_expr.
          after: Tiempo de referencia. El proximo tick sera estrictamente posterior.

      Returns:
          Proximo datetime que satisface el schedule, o None si no existe en 366 dias.
      """
      minutes: list[int] = schedule["minute"]
      hours: list[int] = schedule["hour"]
      doms: list[int] = schedule["day_of_month"]
      months: list[int] = schedule["month"]
      dows: list[int] = schedule["day_of_week"]

      # Truncate to minute and advance by 1.
      t = after.replace(second=0, microsecond=0) + timedelta(minutes=1)
      limit = after + timedelta(days=366)

      while t < limit:
          # Check month.
          if t.month not in months:
              t = _next_valid_month(t, months)
              if t is None:
                  return None
              continue

          # Check day_of_month AND day_of_week (AND semantics).
          if t.day not in doms or t.weekday() not in _python_dows(dows):
              t = t.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1)
              continue

          # Check hour.
          if t.hour not in hours:
              nxt = _next_valid_hour(t, hours)
              if nxt is None:
                  t = t.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=1)
              else:
                  t = nxt
              continue

          # Check minute.
          if t.minute not in minutes:
              nxt = _next_valid_minute(t, minutes)
              if nxt is None:
                  t = t.replace(minute=0, second=0, microsecond=0) + timedelta(hours=1)
              else:
                  t = nxt
              continue

          return t

      return None


  def _python_dows(dows: list[int]) -> set[int]:
      """Convert cron day-of-week (0=Sun) to Python weekday() (0=Mon, 6=Sun)."""
      result = set()
      for d in dows:
          # cron: 0=Sun,1=Mon,...,6=Sat
          # python weekday: 0=Mon,...,6=Sun
          result.add((d - 1) % 7)
      return result


  def _next_valid_month(t: datetime, months: list[int]) -> datetime | None:
      month = t.month
      for m in months:
          if m > month:
              return t.replace(month=m, day=1, hour=0, minute=0, second=0, microsecond=0)
      # Wrap to next year.
      if months:
          try:
              return t.replace(year=t.year + 1, month=months[0], day=1, hour=0, minute=0, second=0, microsecond=0)
          except ValueError:
              return None
      return None


  def _next_valid_hour(t: datetime, hours: list[int]) -> datetime | None:
      h = t.hour
      for hh in hours:
          if hh > h:
              return t.replace(hour=hh, minute=0, second=0, microsecond=0)
      return None


  def _next_valid_minute(t: datetime, minutes: list[int]) -> datetime | None:
      m = t.minute
      for mm in minutes:
          if mm > m:
              return t.replace(minute=mm, second=0, microsecond=0)
      return None

params_schema:
  {"params":[{"name":"schedule","desc":"Diccionario de cron schedule parsado con parse_cron_expr"},{"name":"after","desc":"Datetime base desde el cual calcular la próxima ejecución"}],"output":"Próximo datetime cuando se ejecute el cron, o None si el schedule es imposible"}


id: normalize_entity_name_py_core
name: normalize_entity_name
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def normalize_entity_name(name: str, entity_type: str = "") -> str
tags: ["normalize","entity","name","deduplication","osint","fuzzygraph","nlp","text"]
uses_functions: []
uses_types: []
returns: []
imports: ["re"]
test_file_path: python/functions/core/normalize_entity_name_test.py
file_path: python/functions/core/normalize_entity_name.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 44c807e4d3f7817ed54a0879733921183a4549eaf4ef41cce606b4ee720f6d17
description:
  Normaliza el nombre de una entidad para comparacion y deduplicacion. Aplica reglas distintas segun el tipo: ip/email/domain/crypto_wallet/phone usan normalizacion tecnica; person remueve titulos y convierte formato Apellido-Nombre; organization elimina sufijos legales; default aplica lower+strip+colapso de espacios.

example:
  ```python
  # Persona con titulo y whitespace
  normalize_entity_name("  Dr. Jane Doe  ", "person")   # "Jane Doe"

  # Formato apellido-nombre
  normalize_entity_name("Smith, John", "person")          # "John Smith"

  # Organizacion con sufijo legal
  normalize_entity_name("Google LLC", "organization")     # "Google"
  normalize_entity_name("ACME Corp.", "organization")     # "Acme"

  # Tecnico: dominio con www y trailing dot
  normalize_entity_name("www.example.com.", "domain")     # "example.com"

  # Email case-insensitive
  normalize_entity_name("user@GMAIL.com", "email")        # "user@gmail.com"

  # Telefono: solo digitos y +
  normalize_entity_name("+1 (555) 123-4567", "phone")     # "+15551234567"

  # Crypto wallet es case-sensitive
  normalize_entity_name("1A1zP1eP5QGefi2DMPTfTL5SLmv7Divf", "crypto_wallet")
  # "1A1zP1eP5QGefi2DMPTfTL5SLmv7Divf" (sin cambio de case)

  # Default
  normalize_entity_name("  Hello   WORLD  ")              # "hello world"
  ```

tests:
  ["person strip whitespace","person apellido nombre","person titulo","organization llc","organization corp","ip sin cambio","email lowercase","domain www y trailing dot","phone formato","default lowercase","default colapsa espacios","crypto wallet case sensitive","domain solo trailing dot","person prof titulo"]

notes:
  Disenada para el modulo fuzzygraph donde entidades de distintos tipos necesitan normalizacion previa a la comparacion y deduplicacion.

  El parametro `entity_type` es case-insensitive. Valores soportados: `ip`, `email`, `domain`, `crypto_wallet`, `phone`, `person`, `organization`. Cualquier otro valor (incluido string vacio) activa el comportamiento default.

  Bitcoin y otras crypto addresses son case-sensitive por diseno — el tipo `crypto_wallet` solo aplica strip sin cambiar el case.

  Los titulos de persona reconocidos incluyen: Dr, Mr, Mrs, Ms, Miss, Prof, Sr, Jr, Ing, Lic, Gen, Col, Maj, Capt, Sgt, Rev, Hon (con o sin punto final).

  Los sufijos legales de organizacion reconocidos incluyen: Inc, LLC, Ltd, Corp, Co, SA, GmbH, BV, NV, PLC, AG, SRL, SL, Pty, LP, LLP, LLLP, PC, PA, PLLC, Foundation, Group, Holdings, Enterprises, International, Industries, Services, Solutions, Systems, Technologies.

code:
  """Normaliza el nombre de una entidad para comparacion y deduplicacion."""

  import re


  _TITLES = re.compile(
      r"^\b(?:Dr|Mr|Mrs|Ms|Miss|Prof|Sr|Jr|Ing|Lic|Gen|Col|Maj|Capt|Sgt|Rev|Hon)\.?\s+",
      re.IGNORECASE,
  )

  _LEGAL_SUFFIXES = re.compile(
      r"\b(?:Inc|LLC|Ltd|Corp|Co|S\.?A|GmbH|B\.?V|N\.?V|PLC|AG|SRL|S\.?L|Pty|"
      r"LP|LLP|LLLP|PC|PA|PLLC|Foundation|Group|Holdings|Enterprises?|"
      r"International|Industries|Services?|Solutions?|Systems?|Technologies?)\.?\s*$",
      re.IGNORECASE,
  )

  _MULTI_SPACE = re.compile(r"\s+")


  def normalize_entity_name(name: str, entity_type: str = "") -> str:
      """Normaliza el nombre de una entidad para comparacion y deduplicacion.

      Aplica reglas diferentes segun el tipo de entidad:
      - ip / email / domain / crypto_wallet / phone: normalizacion tecnica
      - person: normalizacion de nombre humano (titulos, formato apellido-nombre)
      - organization: normalizacion corporativa (sufijos legales)
      - default: lower + strip + colapsar espacios

      Args:
          name: nombre de la entidad a normalizar.
          entity_type: tipo de entidad (ip, email, domain, crypto_wallet, phone,
                       person, organization). Vacio = default.

      Returns:
          nombre normalizado como string.
      """
      name = name.strip()
      et = entity_type.lower().strip()

      if et == "ip":
          return name.lower()

      if et == "email":
          return name.lower()

      if et == "domain":
          result = name.lower().rstrip(".")
          if result.startswith("www."):
              result = result[4:]
          return result

      if et == "crypto_wallet":
          # Bitcoin addresses son case-sensitive — solo strip
          return name

      if et == "phone":
          # Mantener solo digitos y el signo +
          return re.sub(r"[^\d+]", "", name)

      if et == "person":
          # Remover titulos al inicio
          result = _TITLES.sub("", name).strip()
          # Detectar formato "Apellido, Nombre"
          if "," in result:
              parts = result.split(",", 1)
              last = parts[0].strip()
              first = parts[1].strip()
              result = f"{first} {last}"
          # Colapsar espacios y title case
          result = _MULTI_SPACE.sub(" ", result).strip()
          return result.title()

      if et == "organization":
          result = _LEGAL_SUFFIXES.sub("", name).strip()
          result = _MULTI_SPACE.sub(" ", result).strip()
          # Title case para consistencia
          return result.title()

      # Default: lower, strip, colapsar espacios
      return _MULTI_SPACE.sub(" ", name.lower()).strip()

params_schema:
  {"params":[{"name":"name","desc":"Nombre de la entidad a normalizar"},{"name":"entity_type","desc":"Tipo de entidad para aplicar reglas específicas (person, organization, email, domain, ip, etc.)"}],"output":"Nombre normalizado según el tipo de entidad"}


id: page_list_to_groups_py_core
name: page_list_to_groups
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
tags: ["chunking","tokens","pagination","grouping"]
uses_functions: []
uses_types: []
returns: []
imports: ["math"]
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 31a28ba9ecaf69f6796c057332980e39425762bfb5854e9099a9b9593355e0db
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/page_index.py
signature:
  def page_list_to_groups(page_contents: list[str], token_lengths: list[int], max_tokens: int = 20000, overlap_pages: int = 1) -> list[str]

description:
  Agrupa paginas en chunks de texto respetando limite de tokens con overlap configurable entre grupos.

example:
  ```python
  contents = ["page1 text", "page2 text", "page3 text", "page4 text"]
  tokens = [5000, 6000, 7000, 4000]
  groups = page_list_to_groups(contents, tokens, max_tokens=12000, overlap_pages=1)
  # 2 groups con overlap de 1 pagina entre ellos
  ```

notes:
  Funcion pura. Si el total de tokens cabe en max_tokens, retorna un solo grupo. El overlap permite contexto compartido entre chunks adyacentes. Util para procesar documentos largos con LLMs.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"page_contents","desc":"Lista de strings con contenido de cada página"},{"name":"token_lengths","desc":"Lista paralela con número de tokens por página"},{"name":"max_tokens","desc":"Límite máximo de tokens por grupo"},{"name":"overlap_pages","desc":"Número de páginas a solapar entre grupos consecutivos"}],"output":"Lista de strings donde cada string es un grupo de páginas combinadas"}


id: parse_code_ast_py_core
name: parse_code_ast
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def parse_code_ast(source_code: str, language: str) -> list[CodeEntity]
tags: ["ast","parsing","tree-sitter","code","symbol","extraction","analysis","multi-language"]
uses_functions: []
uses_types: ["code_entity_py_core"]
returns: ["code_entity_py_core"]
test_file_path: python/functions/core/parse_code_ast_test.py
file_path: python/functions/core/parse_code_ast.py
created_at: 2026-04-05T11:25:56Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: a4c0c0b698cfd32d349598298d55cba7e43dce0a52cb6d06413be12e78b823f1
description:
  Extrae entidades de codigo (funciones, clases, metodos, tipos, interfaces, structs, traits) de codigo fuente usando tree-sitter. Soporta Python, JavaScript, TypeScript, Go, Rust, Java y C++. Opera sobre strings: no accede al disco.

imports:
  ["tree-sitter","tree-sitter-python","tree-sitter-go","tree-sitter-javascript","tree-sitter-typescript","tree-sitter-rust","tree-sitter-java","tree-sitter-cpp"]

example:
  ```python
  import sys
  sys.path.insert(0, "python/functions/core")
  sys.path.insert(0, "python/types/core")

  from parse_code_ast import parse_code_ast

  source = '''
  def greet(name: str) -> str:
      """Saluda al usuario."""
      return f"Hello, {name}!"

  class Animal:
      def speak(self) -> str:
          return ""
  '''

  entities = parse_code_ast(source, "python")
  for e in entities:
      print(e.kind, e.name, f"lines {e.start_line}-{e.end_line}")
      for child in e.children:
          print("  ->", child.kind, child.name)

  # function greet lines 2-4
  # class Animal lines 6-8
  #   -> method speak
  ```

tests:
  ["lenguaje no soportado lanza value error","lenguaje no soportado lanza value error vacio","codigo vacio retorna lista vacia","codigo solo espacios retorna lista vacia","alias py funciona","alias ts funciona","lenguajes soportados incluyen go y python","python extrae funcion simple","python extrae clase con metodos","python extrae docstring","go extrae funciones y tipos","typescript extrae interfaces","codigo con errores de sintaxis es tolerante"]

notes:
  - **Pura sobre strings**: opera exclusivamente sobre `source_code: str`. No accede a disco, red ni estado global. Es deterministicamente pura dada una entrada.
  - **Tolerancia a errores**: tree-sitter parsea incluso codigo con errores de sintaxis, produciendo nodos ERROR en el AST pero continuando el parsing. La funcion retorna lo que pueda extraer.
  - **Imports lazy**: `tree-sitter` y los grammars se importan en el momento de llamar a la funcion, no al importar el modulo. Esto permite importar `parse_code_ast` sin tener tree-sitter instalado.
  - **Firma completa**: la firma incluye decoradores (Python), generic params (Go/TypeScript), visibility modifiers (Rust/Java). Es todo el texto desde el inicio de la definicion hasta el inicio del body.
  - **Children un nivel**: solo se extraen metodos directos de clases. No se hace recursion profunda (un impl de Rust con traits anidados producira children del impl, no sub-children).
  - **API tree-sitter >= 0.22**: usa `Language(mod.language())` en lugar del antiguo `Language.build_library()`. Compatible con los paquetes pip actuales.
  - Reimplementado desde cero. Inspirado conceptualmente en `openviking/parse/parsers/code/ast/` (AGPL-3.0) pero sin copiar codigo.

documentation:
  ## Lenguajes soportados

  | Lenguaje | Alias | Grammar pip | Entidades extraidas |
  |---|---|---|---|
  | `python` | `py` | `tree-sitter-python` | function, class, method |
  | `javascript` | `js` | `tree-sitter-javascript` | function, class, method |
  | `typescript` | `ts` | `tree-sitter-typescript` | function, class, method, interface, type |
  | `go` | — | `tree-sitter-go` | function, method, type |
  | `rust` | `rs` | `tree-sitter-rust` | function, struct, class (impl), trait |
  | `java` | — | `tree-sitter-java` | class, method, interface, function (constructor) |
  | `cpp` | `c++`, `cxx` | `tree-sitter-cpp` | function, class, struct, template |

  ## Algoritmo

  1. Normalizar alias de lenguaje (`py` → `python`, `ts` → `typescript`, etc.)
  2. Importar `tree-sitter` y el grammar del lenguaje (falla con ImportError claro si no instalado)
  3. Construir `Language` desde el modulo del grammar (API tree-sitter >= 0.22)
  4. Parsear `source_code.encode("utf-8")` → AST (tree-sitter tolera errores de sintaxis)
  5. Walk recursivo del AST — para cada nodo relevante:
     - Extraer nombre desde hijo `identifier` / `type_identifier`
     - Extraer firma: texto del nodo hasta el primer nodo body
     - Extraer docstring: primer string literal en Python; comentario `//` / `///` / `/** */` previo en otros lenguajes
     - Para clases/structs/traits/interfaces: extraer metodos como `children`
  6. Retornar lista plana de entidades de nivel superior

  ## Instalacion de dependencias

  ```bash
  # Base (obligatorio)
  pip install tree-sitter

  # Grammars (instalar solo los que necesites)
  pip install tree-sitter-python
  pip install tree-sitter-go
  pip install tree-sitter-javascript
  pip install tree-sitter-typescript
  pip install tree-sitter-rust
  pip install tree-sitter-java
  pip install tree-sitter-cpp
  ```

code:
  """parse_code_ast — extrae entidades de codigo fuente usando tree-sitter."""

  from __future__ import annotations

  import sys
  import os

  # Permite importar CodeEntity desde python/types/core/
  _TYPES_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "..", "python", "types", "core")
  _TYPES_DIR = os.path.abspath(_TYPES_DIR)
  if _TYPES_DIR not in sys.path:
      sys.path.insert(0, _TYPES_DIR)

  try:
      from code_entity import CodeEntity
  except ImportError:
      # Fallback: define localmente para que el modulo pueda importarse
      from dataclasses import dataclass, field

      @dataclass
      class CodeEntity:  # type: ignore[no-redef]
          kind: str
          name: str
          start_line: int
          end_line: int
          signature: str
          docstring: str | None
          language: str
          children: list["CodeEntity"] = field(default_factory=list)


  # ---------------------------------------------------------------------------
  # Lenguajes soportados
  # ---------------------------------------------------------------------------

  SUPPORTED_LANGUAGES: dict[str, str] = {
      "python": "tree_sitter_python",
      "javascript": "tree_sitter_javascript",
      "typescript": "tree_sitter_typescript",
      "go": "tree_sitter_go",
      "rust": "tree_sitter_rust",
      "java": "tree_sitter_java",
      "cpp": "tree_sitter_cpp",
  }

  # Alias normalizados
  _ALIASES: dict[str, str] = {
      "py": "python",
      "js": "javascript",
      "ts": "typescript",
      "rs": "rust",
      "c++": "cpp",
      "cxx": "cpp",
  }

  # Nodos AST relevantes por lenguaje
  _ENTITY_NODES: dict[str, dict[str, str]] = {
      "python": {
          "function_definition": "function",
          "class_definition": "class",
          "decorated_definition": "function",  # resuelto en runtime
      },
      "javascript": {
          "function_declaration": "function",
          "function_expression": "function",
          "arrow_function": "function",
          "class_declaration": "class",
          "method_definition": "method",
      },
      "typescript": {
          "function_declaration": "function",
          "function_expression": "function",
          "arrow_function": "function",
          "class_declaration": "class",
          "method_definition": "method",
          "interface_declaration": "interface",
          "type_alias_declaration": "type",
      },
      "go": {
          "function_declaration": "function",
          "method_declaration": "method",
          "type_declaration": "type",
      },
      "rust": {
          "function_item": "function",
          "struct_item": "struct",
          "impl_item": "class",  # impl block tratado como class con metodos
          "trait_item": "trait",
      },
      "java": {
          "class_declaration": "class",
          "method_declaration": "method",
          "interface_declaration": "interface",
          "constructor_declaration": "function",
      },
      "cpp": {
          "function_definition": "function",
          "class_specifier": "class",
          "template_declaration": "function",
          "struct_specifier": "struct",
      },
  }


  # ---------------------------------------------------------------------------
  # Helpers de extraccion
  # ---------------------------------------------------------------------------

  def _node_text(node, source_bytes: bytes) -> str:
      """Retorna el texto crudo del nodo."""
      return source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")


  def _extract_name(node, source_bytes: bytes, lang: str) -> str:
      """Extrae el nombre de la entidad desde el nodo AST."""
      # Buscar nodo hijo con type 'identifier' o 'name'
      for child in node.children:
          if child.type in ("identifier", "name", "type_identifier", "field_identifier"):
              return _node_text(child, source_bytes)
      # Para decorated_definition en Python, ir al nodo interno
      if node.type == "decorated_definition":
          for child in node.children:
              if child.type in ("function_definition", "class_definition"):
                  return _extract_name(child, source_bytes, lang)
      return "<anonymous>"


  def _extract_signature(node, source_bytes: bytes, lang: str) -> str:
      """Extrae la firma del nodo: todo antes del body."""
      body_types = {
          "block", "body", "statement_block", "compound_statement",
          "declaration_list", "field_declaration_list", "class_body",
          "enum_body", "struct_body", "impl_body",
      }
      # Acumular texto hasta encontrar el body
      start = node.start_byte
      body_start = None
      for child in node.children:
          if child.type in body_types:
              body_start = child.start_byte
              break
      if body_start is not None:
          sig = source_bytes[start:body_start].decode("utf-8", errors="replace").strip()
      else:
          # Sin body distinguible (ej: type alias, interface en algunos grammars)
          sig = _node_text(node, source_bytes).split("\n")[0].strip()
      return sig


  def _extract_docstring_python(node, source_bytes: bytes) -> str | None:
      """Extrae docstring Python: primer string literal en el body."""
      body_types = {"block", "body"}
      for child in node.children:
          if child.type in body_types:
              for stmt in child.children:
                  if stmt.type == "expression_statement":
                      for inner in stmt.children:
                          if inner.type in ("string", "concatenated_string"):
                              raw = _node_text(inner, source_bytes)
                              # Limpiar comillas triples / simples
                              for q in ('"""', "'''", '"', "'"):
                                  if raw.startswith(q) and raw.endswith(q) and len(raw) > 2 * len(q):
                                      return raw[len(q):-len(q)].strip()
                              return raw.strip()
                  # Solo miramos el primer statement real
                  if stmt.type not in ("comment", "\n", "pass_statement"):
                      break
      return None


  def _extract_docstring_comment(node, source_bytes: bytes, all_bytes: bytes) -> str | None:
      """Extrae comentario previo al nodo (para Go, Rust, Java, C++)."""
      # Buscar comentario en siblings anteriores del nodo en el tree
      # Usamos el inicio del nodo y buscamos hacia atras en el texto
      start = node.start_byte
      preceding = all_bytes[:start].decode("utf-8", errors="replace")
      lines = preceding.split("\n")
      # Recolectar lineas de comentario inmediatamente anteriores
      comment_lines: list[str] = []
      for line in reversed(lines):
          stripped = line.strip()
          if stripped.startswith("///") or stripped.startswith("//"):
              comment_lines.insert(0, stripped.lstrip("/").strip())
          elif stripped.startswith("*") or stripped.startswith("/**") or stripped.startswith("*/"):
              content = stripped.lstrip("/*").rstrip("*/").strip()
              if content:
                  comment_lines.insert(0, content)
          elif stripped == "":
              # Linea en blanco: detener busqueda solo si ya tenemos algo
              if comment_lines:
                  break
          else:
              break
      return "\n".join(comment_lines) if comment_lines else None


  def _extract_docstring(node, source_bytes: bytes, lang: str) -> str | None:
      """Extrae docstring/comentario segun el lenguaje."""
      if lang == "python":
          # Para decorated_definition, extraer del nodo interno
          if node.type == "decorated_definition":
              for child in node.children:
                  if child.type in ("function_definition", "class_definition"):
                      return _extract_docstring_python(child, source_bytes)
          return _extract_docstring_python(node, source_bytes)
      else:
          return _extract_docstring_comment(node, source_bytes, source_bytes)


  def _resolve_kind(node_type: str, node, source_bytes: bytes, lang: str, entity_map: dict) -> str:
      """Resuelve el kind de la entidad (maneja decorated_definition en Python)."""
      base_kind = entity_map.get(node_type, "function")
      if node_type == "decorated_definition":
          for child in node.children:
              if child.type == "class_definition":
                  return "class"
              if child.type == "function_definition":
                  return "function"
      return base_kind


  def _extract_children(node, source_bytes: bytes, lang: str, entity_map: dict) -> list[CodeEntity]:
      """Extrae metodos/funciones hijos dentro de una clase/impl/trait."""
      children: list[CodeEntity] = []
      method_kinds = {
          "method_definition", "method_declaration", "function_definition",
          "function_declaration", "function_item",
      }
      body_types = {
          "block", "body", "statement_block", "class_body", "declaration_list",
          "impl_body", "field_declaration_list",
      }

      def _walk_for_methods(n):
          for child in n.children:
              if child.type in body_types:
                  _walk_for_methods(child)
              elif child.type in method_kinds and child.type in entity_map:
                  kind = entity_map[child.type]
                  name = _extract_name(child, source_bytes, lang)
                  start_line = child.start_point[0] + 1
                  end_line = child.end_point[0] + 1
                  sig = _extract_signature(child, source_bytes, lang)
                  doc = _extract_docstring(child, source_bytes, lang)
                  children.append(CodeEntity(
                      kind=kind,
                      name=name,
                      start_line=start_line,
                      end_line=end_line,
                      signature=sig,
                      docstring=doc,
                      language=lang,
                      children=[],
                  ))

      _walk_for_methods(node)
      return children


  # ---------------------------------------------------------------------------
  # Funcion principal
  # ---------------------------------------------------------------------------

  def parse_code_ast(source_code: str, language: str) -> list[CodeEntity]:
      """Extrae entidades de codigo fuente usando tree-sitter.

      Parsea el codigo fuente y retorna una lista plana de entidades de nivel
      superior (funciones, clases, tipos, interfaces). Las clases incluyen sus
      metodos como `children`.

      Args:
          source_code: Codigo fuente como string.
          language: Lenguaje del codigo. Valores soportados: "python",
              "javascript", "typescript", "go", "rust", "java", "cpp".
              Se aceptan alias: "py", "js", "ts", "rs", "c++", "cxx".

      Returns:
          Lista de CodeEntity con las entidades extraidas. Lista vacia si el
          codigo no contiene entidades reconocibles o es vacio.

      Raises:
          ValueError: Si el lenguaje no esta soportado.
          ImportError: Si tree-sitter o el grammar del lenguaje no estan
              instalados. Instalar con:
              `pip install tree-sitter tree-sitter-python tree-sitter-go` etc.
      """
      if not source_code or not source_code.strip():
          return []

      # Normalizar lenguaje
      lang = language.lower().strip()
      lang = _ALIASES.get(lang, lang)

      if lang not in SUPPORTED_LANGUAGES:
          supported = ", ".join(sorted(SUPPORTED_LANGUAGES))
          raise ValueError(
              f"Lenguaje '{language}' no soportado. "
              f"Soportados: {supported}"
          )

      # Importar tree-sitter
      try:
          import tree_sitter  # noqa: F401
          from tree_sitter import Language, Parser
      except ImportError as exc:
          raise ImportError(
              "tree-sitter no esta instalado. Instalar con: "
              "pip install tree-sitter tree-sitter-python tree-sitter-go "
              "tree-sitter-javascript tree-sitter-typescript tree-sitter-rust "
              "tree-sitter-java tree-sitter-cpp"
          ) from exc

      # Importar grammar del lenguaje
      grammar_module = SUPPORTED_LANGUAGES[lang]
      try:
          import importlib
          mod = importlib.import_module(grammar_module)
      except ImportError as exc:
          pip_name = grammar_module.replace("_", "-")
          raise ImportError(
              f"Grammar '{grammar_module}' no instalado. Instalar con: "
              f"pip install {pip_name}"
          ) from exc

      # Construir Language segun la API de tree-sitter >= 0.22
      try:
          # API nueva: el modulo tiene una funcion language() que retorna el Language
          if lang == "typescript":
              # tree-sitter-typescript expone typescript y tsx por separado
              ts_lang = Language(mod.language_typescript())
          elif hasattr(mod, "language"):
              ts_lang = Language(mod.language())
          else:
              # Fallback: algunos grammars exponen Language directamente
              ts_lang = Language(mod.Language)
      except Exception as exc:
          raise ImportError(
              f"No se pudo inicializar el grammar para '{lang}': {exc}. "
              f"Verifica que {grammar_module} es compatible con esta version de tree-sitter."
          ) from exc

      parser = Parser(ts_lang)

      source_bytes = source_code.encode("utf-8")
      tree = parser.parse(source_bytes)

      entity_map = _ENTITY_NODES.get(lang, {})
      top_level_kinds = {"class", "struct", "trait", "interface"}

      results: list[CodeEntity] = []

      def _walk(node, depth: int = 0) -> None:
          """Walk recursivo del AST para extraer entidades de nivel superior."""
          if node.type in entity_map and depth <= 2:
              kind = _resolve_kind(node.type, node, source_bytes, lang, entity_map)
              name = _extract_name(node, source_bytes, lang)
              start_line = node.start_point[0] + 1
              end_line = node.end_point[0] + 1
              sig = _extract_signature(node, source_bytes, lang)
              doc = _extract_docstring(node, source_bytes, lang)

              # Extraer children solo para contenedores
              children: list[CodeEntity] = []
              if kind in top_level_kinds:
                  children = _extract_children(node, source_bytes, lang, entity_map)

              results.append(CodeEntity(
                  kind=kind,
                  name=name,
                  start_line=start_line,
                  end_line=end_line,
                  signature=sig,
                  docstring=doc,
                  language=lang,
                  children=children,
              ))
              # No seguir descendiendo si ya extrajimos la entidad (children ya incluidos)
              return

          for child in node.children:
              _walk(child, depth + 1)

      _walk(tree.root_node)
      return results

params_schema:
  {"params":[{"name":"source_code","desc":"String con el código fuente a analizar"},{"name":"language","desc":"Lenguaje del código (python/py, go, javascript/js, typescript/ts, rust/rs, java, cpp/c++)"}],"output":"Lista de entidades CodeEntity extraídas del AST (funciones, clases, métodos, interfaces)"}


id: parse_cron_expr_py_core
name: parse_cron_expr
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: parse_cron_expr(expr: str) -> dict
tags: ["cron","scheduling","parsing","time","pure"]
uses_functions: []
uses_types: []
returns: []
imports: []
test_file_path: python/functions/core/parse_cron_expr_test.py
file_path: python/functions/core/parse_cron_expr.py
created_at: 2026-04-05T14:43:57Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 8d6c5ba0e54e616c0487cf75b6f6d88f995fd77ffd20938cafa96a2dcc8921b3
description:
  Parsea una expresion cron estandar de 5 campos. Soporta *, rangos (1-5), listas (1,3,5), pasos (*/15) y aliases (@hourly, @daily, @weekly, @monthly, @yearly). Retorna dict con claves minute, hour, day_of_month, month, day_of_week con listas de enteros validos.

example:
  ```python
  from parse_cron_expr import parse_cron_expr

  sched = parse_cron_expr("*/15 * * * *")
  # sched["minute"] = [0, 15, 30, 45]
  # sched["hour"]   = list(range(24))

  sched2 = parse_cron_expr("@daily")
  # sched2["minute"] = [0]
  # sched2["hour"]   = [0]

  sched3 = parse_cron_expr("0 9 * * 1-5")
  # sched3["day_of_week"] = [1, 2, 3, 4, 5]
  ```

tests:
  ["*/15 expande minutos a [0 15 30 45]","@daily resuelve a minuto 0 hora 0","0 9 1,15 * * expande dias a [1 15]","0 9 * * 1-5 expande dia de semana a [1 2 3 4 5]","expresion con 4 campos lanza ValueError","minuto fuera de rango lanza ValueError"]

notes:
  Misma semantica que la version Go (parse_cron_expr_go_core). Usa dict en lugar de struct. Limites de campos: minute [0,59], hour [0,23], day_of_month [1,31], month [1,12], day_of_week [0,6] (0=domingo). Lanza ValueError con mensaje descriptivo para entradas invalidas.

code:
  """Parser de expresiones cron estandar de 5 campos."""

  from __future__ import annotations

  _ALIASES: dict[str, str] = {
      "@yearly": "0 0 1 1 *",
      "@annually": "0 0 1 1 *",
      "@monthly": "0 0 1 * *",
      "@weekly": "0 0 * * 0",
      "@daily": "0 0 * * *",
      "@midnight": "0 0 * * *",
      "@hourly": "0 * * * *",
  }

  # (lo, hi) inclusive for each field: minute, hour, day_of_month, month, day_of_week
  _FIELD_LIMITS: list[tuple[int, int]] = [
      (0, 59),
      (0, 23),
      (1, 31),
      (1, 12),
      (0, 6),
  ]
  _FIELD_NAMES = ["minute", "hour", "day_of_month", "month", "day_of_week"]


  def parse_cron_expr(expr: str) -> dict[str, list[int]]:
      """Parsea una expresion cron estandar de 5 campos.

      Soporta *, rangos (1-5), listas (1,3,5), pasos (*/15) y aliases
      (@hourly, @daily, @weekly, @monthly, @yearly).

      Args:
          expr: Expresion cron de 5 campos o alias.

      Returns:
          Dict con claves minute, hour, day_of_month, month, day_of_week,
          cada una con la lista de enteros validos para ese campo.

      Raises:
          ValueError: Si la expresion es invalida o un valor esta fuera de rango.
      """
      expr = expr.strip()
      expr = _ALIASES.get(expr, expr)
      fields = expr.split()
      if len(fields) != 5:
          raise ValueError(
              f"parse_cron_expr: expected 5 fields, got {len(fields)} in {expr!r}"
          )
      result: dict[str, list[int]] = {}
      for i, field in enumerate(fields):
          lo, hi = _FIELD_LIMITS[i]
          key = _FIELD_NAMES[i]
          result[key] = _parse_field(field, lo, hi, key)
      return result


  def _parse_field(field: str, lo: int, hi: int, name: str) -> list[int]:
      if field == "*":
          return list(range(lo, hi + 1))
      seen: set[int] = set()
      values: list[int] = []
      for part in field.split(","):
          for v in _parse_part(part, lo, hi):
              if v not in seen:
                  seen.add(v)
                  values.append(v)
      values.sort()
      return values


  def _parse_part(part: str, lo: int, hi: int) -> list[int]:
      if "/" in part:
          base, step_str = part.split("/", 1)
          try:
              step = int(step_str)
          except ValueError:
              raise ValueError(f"invalid step {step_str!r}")
          if step <= 0:
              raise ValueError(f"step must be positive, got {step}")
          if base == "*":
              start, end = lo, hi
          elif "-" in base:
              start, end = _parse_range(base, lo, hi)
          else:
              start = _parse_value(base, lo, hi)
              end = hi
          return list(range(start, end + 1, step))
      if "-" in part:
          start, end = _parse_range(part, lo, hi)
          return list(range(start, end + 1))
      return [_parse_value(part, lo, hi)]


  def _parse_range(s: str, lo: int, hi: int) -> tuple[int, int]:
      parts = s.split("-", 1)
      if len(parts) != 2:
          raise ValueError(f"invalid range {s!r}")
      start = _parse_value(parts[0], lo, hi)
      end = _parse_value(parts[1], lo, hi)
      if start > end:
          raise ValueError(f"range start {start} > end {end}")
      return start, end


  def _parse_value(s: str, lo: int, hi: int) -> int:
      try:
          v = int(s)
      except ValueError:
          raise ValueError(f"invalid value {s!r}: not an integer")
      if v < lo or v > hi:
          raise ValueError(f"value {v} out of range [{lo}, {hi}]")
      return v

params_schema:
  {"params":[{"name":"expr","desc":"Expresión cron estándar de 5 campos o alias como @daily, @hourly"}],"output":"Diccionario con claves minute, hour, day_of_month, month, day_of_week conteniendo listas de enteros válidos"}


id: parse_git_url_py_core
name: parse_git_url
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def parse_git_url(url: str, known_hosts: list[str] | None = None) -> str | None
tags: ["git","url","parsing","github","gitlab","repository"]
uses_functions: []
uses_types: []
returns: []
imports: ["urllib.parse"]
test_file_path: python/functions/core/parse_git_url_test.py
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 4ee5e80ade9730ccf6f0a9ba3c7b2713bdc6ac39ab322eceebdca817b27ea0f5
description:
  Parsea una URL de code hosting (GitHub, GitLab, etc.) y retorna el path org/repo. Soporta HTTPS, HTTP, SSH (git@), git:// y ssh://.

example:
  ```python
  parse_git_url("https://github.com/psf/requests")
  # "psf/requests"

  parse_git_url("git@github.com:psf/requests.git")
  # "psf/requests"

  parse_git_url("https://github.com/psf/requests/issues/123")
  # "psf/requests"  (ignora segmentos extra)

  parse_git_url("https://bitbucket.org/org/repo")
  # None  (host desconocido)
  ```

tests:
  ["URL HTTPS GitHub","URL SSH git@","URL con .git suffix","URL con path extra (issues/123)","URL no reconocida"]

notes:
  Funcion pura. Sanitiza org y repo dejando solo `[a-zA-Z0-9_-]`. Los hosts por defecto son `github.com` y `gitlab.com`; se pueden extender via `known_hosts`. No hace requests de red.

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"url","desc":"URL de repositorio de código (HTTPS, SSH, git://, ssh://)"},{"name":"known_hosts","desc":"Lista opcional de hosts conocidos a reconocer"}],"output":"String con formato 'org/repo' si la URL es válida, None si no se puede parsear"}


id: parse_iso_datetime_py_core
name: parse_iso_datetime
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: parse_iso_datetime(value: str) -> datetime
tags: ["datetime","iso8601","parse","time","windows"]
uses_functions: []
uses_types: []
returns: []
imports: ["re","datetime"]
test_file_path: python/functions/core/parse_iso_datetime_test.py
file_path: python/functions/core/parse_iso_datetime.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 9c003264cccefa59999cc793dbfaef318b9fc7b0f01c6853bdc50b1c0a58a823
description:
  Parsea un datetime ISO 8601 tolerando fracciones de segundo con mas de 6 digitos. Windows produce timestamps como 2026-02-21T13:20:23.1470042+08:00 donde la fraccion excede los 6 digitos que acepta datetime.fromisoformat.

example:
  ```python
  from parse_iso_datetime import parse_iso_datetime

  # Timestamp normal
  dt = parse_iso_datetime("2026-02-21T13:20:23+08:00")

  # Timestamp Windows con 7 digitos en fraccion
  dt = parse_iso_datetime("2026-02-21T13:20:23.1470042+08:00")

  # UTC con Z
  dt = parse_iso_datetime("2026-02-21T13:20:23Z")
  ```

tests:
  ["ISO normal sin fraccion","ISO con Z como UTC","ISO con fraccion de mas de 6 digitos","ISO con timezone offset"]

notes:
  Algoritmo:
  1. Regex `(\.\d{6})\d+` trunca la fraccion de segundo a exactamente 6 digitos.
  2. Reemplaza `Z` final por `+00:00` para compatibilidad con `fromisoformat`.
  3. Llama a `datetime.fromisoformat(normalized)`.

  Funcion pura. No hace I/O ni tiene efectos secundarios.

code:
  """Parsea un datetime ISO 8601 tolerando fracciones de segundo con mas de 6 digitos."""

  import re
  from datetime import datetime

  _FRAC_RE = re.compile(r"(\.\d{6})\d+")


  def parse_iso_datetime(value: str) -> datetime:
      """Parsea un datetime ISO 8601 tolerando fracciones de segundo con >6 digitos.

      Windows produce timestamps como ``2026-02-21T13:20:23.1470042+08:00`` donde
      la fraccion de segundo excede los 6 digitos que acepta ``datetime.fromisoformat``.
      Esta funcion trunca la fraccion a 6 digitos antes de parsear.

      Args:
          value: String ISO 8601. Puede contener 'Z' como offset UTC o
                 fracciones de segundo con mas de 6 digitos.

      Returns:
          datetime parseado, con tzinfo si el string incluia offset.
      """
      normalized = _FRAC_RE.sub(r"\1", value)
      normalized = normalized.replace("Z", "+00:00")
      return datetime.fromisoformat(normalized)

params_schema:
  {"params":[{"name":"value","desc":"String en formato ISO 8601 (posiblemente con fracciones de segundo excesivas)"}],"output":"Objeto datetime parseado y normalizado"}


id: parse_llm_json_py_core
name: parse_llm_json
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def parse_llm_json(response: str) -> dict
tags: ["llm","json","parsing","cleaning","think","codeblock"]
uses_functions: ["strip_think_tags_py_core","strip_markdown_codeblock_py_core"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["json"]
test_file_path: python/functions/core/parse_llm_json_test.py
file_path: python/functions/core/parse_llm_json.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: d37286401cf3f83416587c6bf4e06938e6e3224663de3fe1b25da4c7650568fb
description:
  Parsea una respuesta LLM como JSON, limpiando primero think tags y markdown codeblocks. Combina strip_think_tags + strip_markdown_codeblock + json.loads.

example:
  ```python
  # Respuesta limpia
  result = parse_llm_json('{"status": "ok"}')
  # {"status": "ok"}

  # Respuesta con think tags y codeblock
  raw = '<think>I need to return JSON.</think>\n```json\n{"status": "ok"}\n```'
  result = parse_llm_json(raw)
  # {"status": "ok"}

  # JSON invalido lanza ValueError
  parse_llm_json("not json at all")
  # ValueError: parse_llm_json: invalid JSON after cleaning. Cleaned text: 'not json at all'
  ```

tests:
  ["JSON limpio es parseado correctamente","JSON con think tags y codeblock es parseado correctamente","JSON invalido lanza ValueError"]

notes:
  Funcion impura porque puede lanzar ValueError. El mensaje de error incluye el texto ya limpiado (post strip_think_tags y strip_markdown_codeblock) para facilitar el debugging. Composicion de tres funciones: strip_think_tags → strip_markdown_codeblock → json.loads.

code:
  """Parse JSON from LLM responses, cleaning think tags and markdown code blocks."""

  import json

  from strip_think_tags import strip_think_tags
  from strip_markdown_codeblock import strip_markdown_codeblock


  def parse_llm_json(response: str) -> dict:
      """Parse an LLM response as JSON, cleaning think tags and markdown code blocks first.

      Combines strip_think_tags + strip_markdown_codeblock + json.loads into a
      single pipeline. Raises ValueError with the cleaned text included in the
      message to aid debugging.

      Args:
          response: Raw LLM response string.

      Returns:
          Parsed JSON as a dict.

      Raises:
          ValueError: If the cleaned text is not valid JSON.
      """
      cleaned = strip_think_tags(response)
      cleaned = strip_markdown_codeblock(cleaned)
      try:
          return json.loads(cleaned)
      except json.JSONDecodeError as exc:
          raise ValueError(
              f"parse_llm_json: invalid JSON after cleaning. "
              f"Cleaned text: {cleaned!r}"
          ) from exc

params_schema:
  {"params":[{"name":"response","desc":"String con respuesta del LLM (posiblemente con think tags y markdown codeblocks)"}],"output":"Diccionario parseado del JSON limpio"}


id: parse_page_range_py_core
name: parse_page_range
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def parse_page_range(pages: str) -> list[int]
tags: ["parsing","range","pages","string"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Lanza ValueError si un rango tiene start > end.
content_hash: 4d47b364c066da092db85fe81c6fcfa0dc1367cdd27110338e346d246e32a027
source_repo: https://github.com/VectifyAI/PageIndex
source_license: MIT
source_file: pageindex/retrieve.py
description:
  Parsea string de rangos de paginas ('5-7', '3,8', '12') a lista de enteros ordenada y sin duplicados.

example:
  ```python
  parse_page_range("1-3,7,10-12")
  # [1, 2, 3, 7, 10, 11, 12]

  parse_page_range("5")
  # [5]
  ```

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"pages","desc":"String con especificación de páginas (ej: '5-7,3,10-12')"}],"output":"Lista ordenada de números de página sin duplicados"}


id: parser_registry_py_core
name: parser_registry
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
tags: ["parser","registry","plugin","extensible","dispatch","pattern","singleton"]
uses_functions: []
uses_types: ["base_parser_py_core","parse_result_py_core"]
returns: ["parse_result_py_core"]
error_type: error_go_core
imports: []
test_file_path: python/functions/core/parser_registry_test.py
file_path: python/functions/core/parser_registry.py
created_at: 2026-04-05T11:25:06Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: cd508c918240be87ff06ece09e27f1c82b70106995dee9e41371f5eaed31ad86
params_schema: {"output":"ParseResult con contenido parseado y metadatos del parsing"}
signature:
  class ParserRegistry: register(name: str, parser: BaseParser) -> None; unregister(name: str) -> None; get_parser(name: str) -> BaseParser | None; get_parser_for_file(path: str) -> BaseParser | None; async parse(source: str, **kwargs) -> ParseResult; list_parsers() -> list[str]; list_supported_extensions() -> list[str]

description:
  Registry extensible que despacha parsing de archivos al parser correcto basado en extension. Patron plugin: registrar parsers por nombre y extensiones, resolver automaticamente. Mantiene estado mutable (mapa extension→parser). Singleton global disponible via get_registry().

example:
  ```python
  from parser_registry import ParserRegistry, get_registry

  # Usar instancia propia
  registry = ParserRegistry()
  registry.register("markdown", MarkdownParser())
  registry.register("pdf", PDFParser())
  registry.register("text", TextParser())

  result = await registry.parse("documento.md")
  result2 = await registry.parse("Contenido directo\ncon varias lineas")

  # Usar singleton global
  global_registry = get_registry()
  global_registry.register("markdown", MarkdownParser())
  ```

tests:
  ["registrar parser custom","resolver parser por extension de archivo","parsear archivo existente con extension registrada","parsear string de contenido sin path","desregistrar parser elimina nombre y extensiones","extension desconocida hace fallback al parser text"]

notes:
  Patron plugin extensible: cada parser se registra por nombre y declara sus extensiones. El mapa extension→nombre se construye automaticamente al registrar. Unregister limpia el mapa solo si el parser registrado sigue siendo el mismo (evita colisiones si dos parsers comparten extension).

  El singleton `get_registry()` usa lazy initialization thread-unsafe — suficiente para la mayoria de casos. Para uso multi-threaded, crear instancias propias de ParserRegistry.

  Los parsers especiales `"text"`, `"directory"` y `"code"` son convencion — el registry no los crea automaticamente, deben registrarse explicitamente.

documentation:
  ## Algoritmo de despacho

  `parse(source)` sigue este orden de decision:

  1. Si `source` es URL de repositorio de codigo (github.com, gitlab.com, bitbucket.org) → delega al parser `"code"`.
  2. Si `source` parece path de archivo (longitud ≤ 512, sin newlines) y existe en disco:
     - Si es directorio → delega al parser `"directory"`.
     - Resuelve parser por extension → delega.
     - Sin parser para la extension → fallback al parser `"text"` via `parse_content`.
  3. Si no parece path (contenido largo o con newlines) → parsea como string con parser `"text"` via `parse_content`.

code:
  """ParserRegistry — registry extensible que despacha parsing por extension de archivo."""

  from __future__ import annotations

  import re
  from pathlib import Path
  from typing import TYPE_CHECKING

  if TYPE_CHECKING:
      from base_parser import BaseParser
      from parse_result import ParseResult

  # Patron para detectar URLs de repositorios de codigo.
  _CODE_REPO_PATTERN = re.compile(
      r"^https?://(github\.com|gitlab\.com|bitbucket\.org)/[\w.\-]+/[\w.\-]+"
  )

  # Longitud maxima que puede tener un path de archivo valido (heuristica).
  _MAX_PATH_LEN = 512


  def _looks_like_path(source: str) -> bool:
      """Heuristica: True si source parece un path de archivo, no contenido."""
      return len(source) <= _MAX_PATH_LEN and "\n" not in source


  class ParserRegistry:
      """Registry extensible que despacha parsing al parser correcto segun extension.

      Patron plugin: registrar parsers por nombre y lista de extensiones,
      resolver automaticamente al parsear un archivo. El registry mantiene
      un mapa de extension→nombre para O(1) lookup.

      Uso tipico:
          registry = ParserRegistry()
          registry.register("markdown", MarkdownParser())
          registry.register("pdf", PDFParser())
          result = await registry.parse("doc.md")
      """

      def __init__(self) -> None:
          # nombre → instancia de parser
          self._parsers: dict[str, "BaseParser"] = {}
          # extension → nombre del parser
          self._ext_map: dict[str, str] = {}

      # ------------------------------------------------------------------
      # Registro
      # ------------------------------------------------------------------

      def register(self, name: str, parser: "BaseParser") -> None:
          """Registra un parser con el nombre dado.

          Sobreescribe cualquier parser previamente registrado con el mismo nombre.
          Actualiza el mapa de extensiones automaticamente.

          Args:
              name:   Nombre unico del parser (ej: "markdown", "pdf").
              parser: Instancia de BaseParser a registrar.
          """
          self._parsers[name] = parser
          for ext in parser.supported_extensions:
              self._ext_map[ext.lower()] = name

      def unregister(self, name: str) -> None:
          """Elimina un parser del registry.

          Tambien elimina sus extensiones del mapa. Si el nombre no existe,
          no hace nada.

          Args:
              name: Nombre del parser a eliminar.
          """
          parser = self._parsers.pop(name, None)
          if parser is None:
              return
          for ext in parser.supported_extensions:
              key = ext.lower()
              if self._ext_map.get(key) == name:
                  del self._ext_map[key]

      # ------------------------------------------------------------------
      # Resolucion
      # ------------------------------------------------------------------

      def get_parser(self, name: str) -> "BaseParser | None":
          """Retorna el parser registrado bajo el nombre dado.

          Args:
              name: Nombre del parser.

          Returns:
              Instancia de BaseParser o None si no existe.
          """
          return self._parsers.get(name)

      def get_parser_for_file(self, path: str) -> "BaseParser | None":
          """Resuelve el parser correcto para un archivo dado su extension.

          Args:
              path: Path del archivo (puede ser relativo o absoluto).

          Returns:
              Instancia de BaseParser o None si no hay parser para esa extension.
          """
          ext = Path(path).suffix.lower()
          name = self._ext_map.get(ext)
          if name is None:
              return None
          return self._parsers.get(name)

      # ------------------------------------------------------------------
      # Parseo
      # ------------------------------------------------------------------

      async def parse(self, source: str, **kwargs) -> "ParseResult":
          """Parsea source despachando al parser correcto segun el tipo de entrada.

          Algoritmo de despacho:
          1. Si source es URL de repo de codigo → delega al parser "code".
          2. Si source parece path de archivo (corto, sin newlines) y existe en disco:
             - Si es directorio → delega al parser "directory".
             - Resuelve parser por extension → delega.
             - Si no hay parser para la extension → fallback al parser "text".
          3. Si no parece path (contenido largo o con newlines) → parsea como
             string de contenido con el parser "text".

          Args:
              source:   Path de archivo, directorio, URL o string de contenido.
              **kwargs: Argumentos adicionales pasados al parser seleccionado.

          Returns:
              ParseResult con el arbol de nodos del documento.

          Raises:
              ValueError: Si el parser requerido no esta registrado.
          """
          # 1. URL de repositorio de codigo
          if _CODE_REPO_PATTERN.match(source):
              return await self._dispatch("code", source, **kwargs)

          # 2. Parece un path de archivo
          if _looks_like_path(source):
              p = Path(source)
              if p.exists():
                  if p.is_dir():
                      return await self._dispatch("directory", source, **kwargs)
                  parser = self.get_parser_for_file(source)
                  if parser is not None:
                      return await parser.parse(source, **kwargs)
                  # Fallback a text si no hay parser para la extension
                  return await self._dispatch_content("text", source, **kwargs)

          # 3. Es contenido directo (string largo o con newlines)
          text_parser = self._parsers.get("text")
          if text_parser is None:
              raise ValueError(
                  "No hay parser 'text' registrado. Registra un TextParser primero."
              )
          return await text_parser.parse_content(source, **kwargs)

      # ------------------------------------------------------------------
      # Introspeccion
      # ------------------------------------------------------------------

      def list_parsers(self) -> list[str]:
          """Retorna los nombres de todos los parsers registrados.

          Returns:
              Lista de nombres de parsers, ordenada alfabeticamente.
          """
          return sorted(self._parsers.keys())

      def list_supported_extensions(self) -> list[str]:
          """Retorna todas las extensiones soportadas por parsers registrados.

          Returns:
              Lista de extensiones (con punto, lowercase), ordenada alfabeticamente.
          """
          return sorted(self._ext_map.keys())

      # ------------------------------------------------------------------
      # Helpers internos
      # ------------------------------------------------------------------

      async def _dispatch(self, name: str, source: str, **kwargs) -> "ParseResult":
          """Despacha a un parser por nombre via parse()."""
          parser = self._parsers.get(name)
          if parser is None:
              raise ValueError(
                  f"Parser '{name}' no esta registrado. "
                  f"Parsers disponibles: {self.list_parsers()}"
              )
          return await parser.parse(source, **kwargs)

      async def _dispatch_content(
          self, name: str, content: str, **kwargs
      ) -> "ParseResult":
          """Despacha a un parser por nombre via parse_content()."""
          parser = self._parsers.get(name)
          if parser is None:
              raise ValueError(
                  f"Parser '{name}' no esta registrado. "
                  f"Parsers disponibles: {self.list_parsers()}"
              )
          return await parser.parse_content(content, **kwargs)


  # ------------------------------------------------------------------
  # Singleton global
  # ------------------------------------------------------------------

  _registry: ParserRegistry | None = None


  def get_registry() -> ParserRegistry:
      """Retorna la instancia global del ParserRegistry (lazy-initialized).

      Returns:
          Instancia singleton de ParserRegistry.
      """
      global _registry
      if _registry is None:
          _registry = ParserRegistry()
      return _registry


id: partition_py_core
name: partition
kind: function
lang: py
domain: core
version: 1.0.0
purity: pure
signature: def partition(xs: list, pred: callable) -> tuple
description: Divide una lista en dos: (elementos que cumplen el predicado, elementos que no).
tags: ["list","functional","partition","split","python"]
uses_functions: []
uses_types: []
returns: []
imports: []
tests: []
file_path: python/functions/core/core.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
notes: Funcion pura. Retorna tupla de dos listas: (matches, non_matches).
content_hash: afde859b97ca114c587756d63bd8a7005c92729e49dad6d950e2ac7eee83446f
example:
  ```python
  evens, odds = partition([1, 2, 3, 4, 5], lambda n: n % 2 == 0)
  # evens = [2, 4], odds = [1, 3, 5]
  ```

code:
[cell repeats an earlier one in this file: 25395 characters]

params_schema:
  {"params":[{"name":"xs","desc":"Lista a dividir"},{"name":"pred","desc":"Función predicado que retorna True/False"}],"output":"Tupla (elementos_que_cumplen, elementos_que_no_cumplen)"}


id: pdf_to_markdown_py_core
name: pdf_to_markdown
kind: function
lang: py
domain: core
version: 1.0.0
purity: impure
signature: def pdf_to_markdown(pdf_path: str, heading_detection: str = 'auto') -> tuple[str, dict]
tags: ["pdf","markdown","conversion","headings","tables","pdfplumber","parsing"]
uses_types: []
returns: []
error_type: error_go_core
imports: ["pdfplumber","os","sys"]
tests: []
file_path: python/functions/core/pdf_to_markdown.py
created_at: 2026-04-05T11:22:32Z
updated_at: 2026-04-05T16:40:39Z
props: []
emits: []
variant: []
content_hash: 440e3d88d63ac293d2815e1a880ce883a697d7ade029a6702c75c23142b003d5
description:
  Convierte un PDF a markdown. Extrae texto, tablas e inyecta headings detectados desde bookmarks o analisis de fuentes. Retorna (markdown_content, metadata_dict).

uses_functions:
  ["extract_pdf_bookmarks_py_core","detect_headings_by_font_py_core","format_table_to_markdown_py_core"]

example:
  ```python
  from pdf_to_markdown import pdf_to_markdown

  md, meta = pdf_to_markdown("report.pdf")
  print(f"Pages: {meta['pages']}, Headings: {meta['headings_found']}, Tables: {meta['tables_found']}")
  print(md[:500])

  # Con estrategia especifica
  md, meta = pdf_to_markdown("report.pdf", heading_detection="font")
  print(f"Strategy used: {meta['heading_strategy']}")
  ```

notes:
  Estrategias de heading_detection:
  - `"bookmarks"`: extrae outlines/bookmarks del PDF via pdfplumber. Rapido y preciso cuando el PDF tiene bookmarks bien formados.
  - `"font"`: analiza distribucion de font sizes para detectar headings. Util para PDFs sin bookmarks.
  - `"auto"`: intenta bookmarks primero; si no hay, usa analisis de font. Estrategia por defecto.

  El metadata_dict retornado contiene: `pages` (int), `heading_strategy` (str: "bookmarks"|"font"|"none"), `headings_found` (int), `tables_found` (int).

  Tablas se extraen via `page.extract_tables()` de pdfplumber y se formatean como tablas markdown. Requiere `pip install pdfplumber`.

code:
  """Convert a PDF file to markdown, with heading detection and table extraction."""

  import os
  import sys

  import pdfplumber

  # Allow import from same directory when run directly
  _DIR = os.path.dirname(os.path.abspath(__file__))
  if _DIR not in sys.path:
      sys.path.insert(0, _DIR)

  from extract_pdf_bookmarks import extract_pdf_bookmarks
  from detect_headings_by_font import detect_headings_by_font
  from format_table_to_markdown import format_table_to_markdown


  def pdf_to_markdown(
      pdf_path: str,
      heading_detection: str = "auto",
  ) -> tuple[str, dict]:
      """Convert a PDF file to markdown.

      Extracts text, tables, and injects detected headings from bookmarks or
      font size analysis.

      Args:
          pdf_path: Path to the PDF file.
          heading_detection: Strategy for detecting headings:
              - "bookmarks": use PDF outline/bookmarks only.
              - "font": analyze font size distribution.
              - "auto": try bookmarks first, fall back to font analysis.

      Returns:
          tuple[str, dict]: (markdown_content, metadata_dict).
              metadata_dict contains: {"pages": int, "heading_strategy": str,
              "headings_found": int, "tables_found": int}

      Raises:
          FileNotFoundError: If pdf_path does not exist.
          Exception: If the PDF cannot be opened or is corrupted.
      """
      if not os.path.exists(pdf_path):
          raise FileNotFoundError(f"PDF not found: {pdf_path}")

      with pdfplumber.open(pdf_path) as pdf:
          total_pages = len(pdf.pages)

          # Step 1: Detect headings
          headings: list[dict] = []
          strategy_used = heading_detection

          if heading_detection in ("bookmarks", "auto"):
              headings = extract_pdf_bookmarks(pdf)
              if headings:
                  strategy_used = "bookmarks"
              elif heading_detection == "auto":
                  headings = detect_headings_by_font(pdf)
                  strategy_used = "font" if headings else "none"
          elif heading_detection == "font":
              headings = detect_headings_by_font(pdf)
              strategy_used = "font"

          # Step 2: Group headings by page number
          headings_by_page: dict[int, list[dict]] = {}
          for h in headings:
              pn = h.get("page_num")
              if pn is not None:
                  headings_by_page.setdefault(pn, []).append(h)

          # Step 3: Build markdown page by page
          sections: list[str] = []
          tables_found = 0

          for page_idx, page in enumerate(pdf.pages):
              page_num = page_idx + 1
              page_parts: list[str] = []

              # Inject headings for this page
              for h in headings_by_page.get(page_num, []):
                  level = h["level"]
                  title = h["title"]
                  page_parts.append(f"{'#' * level} {title}")

              # Extract page text
              text = page.extract_text() or ""
              if text.strip():
                  page_parts.append(text.strip())

              # Extract tables
              try:
                  tables = page.extract_tables()
              except Exception:
                  tables = []

              for table in tables:
                  if not table:
                      continue
                  # Convert all cells to strings, treating None as empty
                  str_table = [
                      [str(cell) if cell is not None else "" for cell in row]
                      for row in table
                  ]
                  md_table = format_table_to_markdown(str_table)
                  if md_table:
                      page_parts.append(md_table)
                      tables_found += 1

              if page_parts:
                  sections.append("\n\n".join(page_parts))

          markdown_content = "\n\n".join(sections)

          metadata = {
              "pages": total_pages,
              "heading_strategy": strategy_used,
              "headings_found": len(headings),
              "tables_found": tables_found,
          }

          return markdown_content, metadata

params_schema:
  {"params":[{"name":"pdf_path","desc":"Ruta al archivo PDF a convertir"},{"name":"heading_detection","desc":"Estrategia de detección de headings: 'auto', 'bookmarks', 'font'"}],"output":"Tupla (markdown_content, metadata_dict) con el PDF convertido a markdown y metadatos"}


