feat: media_analytics — ETL PC+VPS → ClickHouse + Grafana
2 ETLs cada 5min suben snapshots (Jellyfin, *arr, Prowlarr, gnula, popelis users/mylist/events) a ClickHouse en el VPS, visualizado en Grafana (grafana.datardos.com). Ingesta PC via tunel SSH; popelis via ETL local en el VPS. Usa clickhouse_insert_rows_py_infra.
This commit is contained in:
Egutierrez
@@ -0,0 +1,173 @@
|
||||
-- Esquema analitico media stack. Todas las tablas de snapshot llevan snapshot_ts
|
||||
-- (momento de la captura del ETL, cada 5min) → permite analisis temporal del estado.
|
||||
-- Las tablas de eventos llevan event_ts (instante real del evento).
|
||||
-- Engine MergeTree, particion mensual, orden por (snapshot_ts, clave).
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS analytics;
|
||||
|
||||
-- ============ JELLYFIN ============
|
||||
-- Catalogo: peliculas/series/episodios visibles en la biblioteca.
|
||||
CREATE TABLE IF NOT EXISTS analytics.jellyfin_items (
|
||||
snapshot_ts DateTime,
|
||||
item_id String,
|
||||
type LowCardinality(String), -- Movie | Series | Episode
|
||||
name String,
|
||||
production_year Int32,
|
||||
runtime_min Float32,
|
||||
genres Array(String),
|
||||
community_rating Float32,
|
||||
official_rating String,
|
||||
series_name String,
|
||||
library String,
|
||||
path String,
|
||||
date_created DateTime DEFAULT toDateTime(0)
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, type, item_id);
|
||||
|
||||
-- Usuarios Jellyfin (espejos popelis).
|
||||
CREATE TABLE IF NOT EXISTS analytics.jellyfin_users (
|
||||
snapshot_ts DateTime,
|
||||
user_id String,
|
||||
name String,
|
||||
last_login DateTime DEFAULT toDateTime(0),
|
||||
last_activity DateTime DEFAULT toDateTime(0),
|
||||
is_admin UInt8
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, user_id);
|
||||
|
||||
-- Estado de reproduccion por usuario+item (playcount, visto, ultima vez).
|
||||
CREATE TABLE IF NOT EXISTS analytics.jellyfin_user_items (
|
||||
snapshot_ts DateTime,
|
||||
user_id String,
|
||||
user_name String,
|
||||
item_id String,
|
||||
item_name String,
|
||||
type LowCardinality(String),
|
||||
played UInt8,
|
||||
play_count Int32,
|
||||
playback_pct Float32,
|
||||
last_played DateTime DEFAULT toDateTime(0)
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, user_id, item_id);
|
||||
|
||||
-- Sesiones activas (lo que se esta viendo en el momento del snapshot).
|
||||
CREATE TABLE IF NOT EXISTS analytics.jellyfin_sessions (
|
||||
snapshot_ts DateTime,
|
||||
user_name String,
|
||||
item_name String,
|
||||
item_type LowCardinality(String),
|
||||
client String,
|
||||
device String,
|
||||
play_method String,
|
||||
is_paused UInt8,
|
||||
position_pct Float32
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, user_name);
|
||||
|
||||
-- ============ SCRAPERS TORRENTS (*arr) ============
|
||||
-- Historial Radarr/Sonarr: grabs, imports, fallos.
|
||||
CREATE TABLE IF NOT EXISTS analytics.arr_history (
|
||||
snapshot_ts DateTime,
|
||||
app LowCardinality(String), -- radarr | sonarr
|
||||
history_id Int64,
|
||||
event_type LowCardinality(String), -- grabbed | downloadFolderImported | ...
|
||||
title String,
|
||||
source_title String,
|
||||
indexer String,
|
||||
download_client String,
|
||||
quality String,
|
||||
languages Array(String),
|
||||
event_date DateTime DEFAULT toDateTime(0)
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, app, history_id);
|
||||
|
||||
-- Cola activa de descargas.
|
||||
CREATE TABLE IF NOT EXISTS analytics.arr_queue (
|
||||
snapshot_ts DateTime,
|
||||
app LowCardinality(String),
|
||||
title String,
|
||||
status String,
|
||||
tracked_status String,
|
||||
size_bytes Int64,
|
||||
sizeleft_bytes Int64,
|
||||
timeleft String,
|
||||
indexer String,
|
||||
download_client String
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, app, title);
|
||||
|
||||
-- Indexers Prowlarr: estado + contadores grab/query.
|
||||
CREATE TABLE IF NOT EXISTS analytics.prowlarr_indexers (
|
||||
snapshot_ts DateTime,
|
||||
indexer_id Int32,
|
||||
name String,
|
||||
enable UInt8,
|
||||
protocol String,
|
||||
privacy String,
|
||||
num_grabs Int64,
|
||||
num_queries Int64,
|
||||
num_grab_fail Int64,
|
||||
num_query_fail Int64
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, indexer_id);
|
||||
|
||||
-- ============ GNULA SCRAPPER ============
|
||||
-- Catalogo de pelis en castellano detectadas (gnula_catalog.db).
|
||||
CREATE TABLE IF NOT EXISTS analytics.gnula_movies (
|
||||
snapshot_ts DateTime,
|
||||
href String,
|
||||
title String,
|
||||
year Int32,
|
||||
flags String,
|
||||
lang_es UInt8,
|
||||
status LowCardinality(String), -- pending | downloaded | failed | have
|
||||
in_library UInt8,
|
||||
detected_at String,
|
||||
downloaded_at String
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, href);
|
||||
|
||||
-- ============ POPELIS ============
|
||||
-- Usuarios (estado).
|
||||
CREATE TABLE IF NOT EXISTS analytics.popelis_users (
|
||||
snapshot_ts DateTime,
|
||||
user_id Int64,
|
||||
username String,
|
||||
jf_user_id String,
|
||||
created_at DateTime DEFAULT toDateTime(0)
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, user_id);
|
||||
|
||||
-- Mi lista por usuario (estado).
|
||||
CREATE TABLE IF NOT EXISTS analytics.popelis_mylist (
|
||||
snapshot_ts DateTime,
|
||||
user_id Int64,
|
||||
item_id String,
|
||||
added_at DateTime DEFAULT toDateTime(0)
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY toYYYYMM(snapshot_ts)
|
||||
ORDER BY (snapshot_ts, user_id, item_id);
|
||||
|
||||
-- Eventos (logins, plays, mylist add/remove) — instrumentados en popelis-api.
|
||||
-- Tabla de hechos: dedup por event_id con ReplacingMergeTree.
|
||||
CREATE TABLE IF NOT EXISTS analytics.popelis_events (
|
||||
event_id Int64,
|
||||
event_ts DateTime,
|
||||
user_id Int64,
|
||||
username String,
|
||||
event_type LowCardinality(String), -- login | logout | play | mylist_add | mylist_remove
|
||||
item_id String,
|
||||
meta String,
|
||||
ingested_at DateTime DEFAULT now()
|
||||
) ENGINE = ReplacingMergeTree(ingested_at)
|
||||
PARTITION BY toYYYYMM(event_ts)
|
||||
ORDER BY (event_id);
|
||||
Reference in New Issue
Block a user