349 lines
18 KiB
Python
Executable File
349 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Shrink shows and movies that Sonarr/Radarr have on the "Standard" or "Mini" profile.
|
|
|
|
Demoting something is just picking one of those profiles. Each night this job first asks
|
|
Sonarr/Radarr to search a few titles it hasn't searched before. Both profiles upgrade only
|
|
toward small releases, so the search swaps whatever big files have a small release for
|
|
bandwidth instead of GPU. A week after a title's search, the job re-encodes whatever is
|
|
still big to HEVC on the GPU (NVENC) at the tier's bitrate, verifies the result, swaps it in
|
|
place of the original, and has Sonarr/Radarr rescan. Plex and Jellyfin pick up changes
|
|
through their own folder watchers.
|
|
|
|
Moving something back up means switching it to HD and searching, which redownloads it at
|
|
full size. See README.md.
|
|
|
|
Usage:
|
|
compact-tv.py [--until HH:MM] [--series TITLE] [--limit N] [--dry-run]
|
|
|
|
--series matches a show or movie title. Runs as gib (files are gib-owned) from
|
|
compact-tv.service, nightly.
|
|
"""
|
|
import argparse, datetime, fcntl, json, os, pathlib, re, shutil, subprocess, sys, time, urllib.request
|
|
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
STATE = HERE / "state.json"
|
|
LOG = HERE / "compact-tv.log"
|
|
WORK = pathlib.Path.home() / ".cache" / "compact-tv" # NVMe, so half-done encodes never land in a ZFS snapshot
|
|
VOLUMES = pathlib.Path.home() / "Server/Media/volumes"
|
|
|
|
# Per app: API port, container path -> host path, and how to list one title's files.
|
|
APPS = {
|
|
"sonarr": {"port": 8989, "paths": ("/tv/", "/home/gib/Public/TV/"), "titles": "/series",
|
|
"files": lambda call, t: call(f"/episodefile?seriesId={t['id']}"),
|
|
"rescan": lambda t: {"name": "RescanSeries", "seriesId": t["id"]}},
|
|
"radarr": {"port": 7878, "paths": ("/movies/", "/home/gib/Public/Movies/"), "titles": "/movie",
|
|
"files": lambda call, t: [t["movieFile"]] if t.get("movieFile") else [],
|
|
"rescan": lambda t: {"name": "RescanMovie", "movieId": t["id"]}},
|
|
}
|
|
|
|
# Video bitrate (kbps) per output class, and the widest output each tier allows.
|
|
# TV floors come from watching samples: 1200 kbps at 1080p, 800 at 720p, 600 at SD.
|
|
# Movies get roughly double: dark scenes, grain and action break down first at low rates,
|
|
# and a 2-hour movie at 2500 kbps is still ~2.3 GB (a quarter of a typical 1080p file).
|
|
# That also leaves already-small releases (YTS-style, ~1.5-2 GB) under the 30% bar.
|
|
# Nothing is ever upscaled: a 720p or SD source keeps its size and uses that class's rate.
|
|
TIERS = {
|
|
"Standard": {"max_width": 1920, "kbps": {"sonarr": {"1080": 1200, "720": 1200, "sd": 600},
|
|
"radarr": {"1080": 2500, "720": 1800, "sd": 900}}},
|
|
"Mini": {"max_width": 1280, "kbps": {"sonarr": {"1080": 800, "720": 800, "sd": 600},
|
|
"radarr": {"1080": 1500, "720": 1500, "sd": 900}}},
|
|
}
|
|
MIN_SAVING = 0.30 # leave a file alone unless re-encoding saves at least this much
|
|
MAX_FAILURES = 2 # after this many failed attempts a file is skipped for good
|
|
|
|
# Search first, encode later. Standard/Mini upgrade only toward small releases (see
|
|
# setup-small-profiles.py), so asking the *arr to search a title swaps whatever big files
|
|
# have a small release, for bandwidth instead of GPU. Each night searches a few titles not
|
|
# searched before, biggest first, so downloads and indexer use stay spread out. A title's
|
|
# files are only encoded once its search is SEARCH_GRACE_DAYS old.
|
|
#
|
|
# Files that arrive after their title's search (new episodes, fresh requests) were never
|
|
# searched for a small copy, so the job searches that episode or movie on its own first,
|
|
# and encodes it only if it is still big FILE_SEARCH_GRACE_DAYS later. Downloads are never
|
|
# delayed by any of this; only encoding waits.
|
|
SEARCHES_PER_NIGHT = {"sonarr": 5, "radarr": 20}
|
|
SEARCH_GRACE_DAYS = 7
|
|
FILE_SEARCH_GRACE_DAYS = 3 # small encodes usually appear within hours to a day; 3 leaves room
|
|
HDR_TRANSFERS = {"smpte2084", "arib-std-b67"}
|
|
|
|
|
|
def log(msg):
|
|
line = f"{datetime.datetime.now():%Y-%m-%d %H:%M:%S} {msg}"
|
|
print(line, flush=True)
|
|
with open(LOG, "a") as f:
|
|
f.write(line + "\n")
|
|
|
|
|
|
# ── Sonarr / Radarr ─────────────────────────────────────────────────────────
|
|
def arr(app):
|
|
"""Return a call(path, body=None) for the app's v3 API."""
|
|
key = re.search(r"<ApiKey>([^<]+)", (VOLUMES / app / "config.xml").read_text()).group(1)
|
|
ip = subprocess.run(["docker", "inspect", app, "--format",
|
|
"{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}"],
|
|
capture_output=True, text=True, check=True).stdout.split()[0]
|
|
base = f"http://{ip}:{APPS[app]['port']}/api/v3"
|
|
|
|
def call(path, body=None):
|
|
req = urllib.request.Request(base + path, headers={"X-Api-Key": key, "Content-Type": "application/json"},
|
|
data=json.dumps(body).encode() if body is not None else None)
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
return json.load(r)
|
|
return call
|
|
|
|
|
|
# ── probing and planning ────────────────────────────────────────────────────
|
|
def probe(path):
|
|
out = subprocess.run(["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", str(path)],
|
|
capture_output=True, text=True)
|
|
return json.loads(out.stdout) if out.returncode == 0 else None
|
|
|
|
|
|
def subtitles(info):
|
|
"""Subtitle tracks ffmpeg can carry over. Tracks ffprobe can't identify are dropped."""
|
|
return [s for s in info["streams"] if s["codec_type"] == "subtitle" and s.get("codec_name")]
|
|
|
|
|
|
def audio_kbps(channels):
|
|
return 96 if channels <= 2 else 160 if channels <= 6 else 192
|
|
|
|
|
|
def plan(info, tier, app):
|
|
"""Return (ffmpeg args, estimated output bytes) for one file, or (None, reason) to skip it."""
|
|
t = TIERS[tier]
|
|
video = next((s for s in info["streams"] if s["codec_type"] == "video"
|
|
and not s.get("disposition", {}).get("attached_pic")), None)
|
|
if not video:
|
|
return None, "no video stream"
|
|
if video.get("color_transfer") in HDR_TRANSFERS or any("Dolby Vision" in str(sd) for sd in video.get("side_data_list", [])):
|
|
return None, "HDR/Dolby Vision source (not handled yet)"
|
|
if (info["format"].get("tags") or {}).get("COMPACT_TIER") == tier:
|
|
return None, f"already {tier}"
|
|
|
|
src_w = int(video["width"])
|
|
out_w = min(src_w, t["max_width"])
|
|
cls = "1080" if out_w >= 1700 else "720" if out_w >= 1200 else "sd"
|
|
vkbps = t["kbps"][app][cls]
|
|
duration = float(info["format"]["duration"])
|
|
audios = [s for s in info["streams"] if s["codec_type"] == "audio"]
|
|
est = int((vkbps + sum(audio_kbps(int(a.get("channels") or 2)) for a in audios)) * 1000 / 8 * duration * 1.03)
|
|
if int(info["format"]["size"]) * (1 - MIN_SAVING) < est:
|
|
return None, f"already small ({int(info['format']['size']) / 1e9 * 3600 / duration:.2f} GB/h)"
|
|
# A downloaded HEVC file within 2x the target is a small release that already replaced a
|
|
# big one. Re-encoding it would cost GPU and quality for little space. Our own outputs
|
|
# (tagged) still go through, so moving a title from Standard to Mini works.
|
|
tags = info["format"].get("tags") or {}
|
|
if video.get("codec_name") == "hevc" and "COMPACT_TIER" not in tags and int(info["format"]["size"]) <= 2 * est:
|
|
return None, f"small HEVC release ({int(info['format']['size']) / 1e9 * 3600 / duration:.2f} GB/h)"
|
|
|
|
vf = []
|
|
if video.get("field_order") not in (None, "progressive", "unknown"):
|
|
vf.append("bwdif")
|
|
if out_w < src_w:
|
|
vf.append(f"scale={out_w}:-2:flags=lanczos")
|
|
args = ["-map", f"0:{video['index']}"]
|
|
if vf:
|
|
args += ["-filter:v", ",".join(vf)]
|
|
args += ["-c:v", "hevc_nvenc", "-preset", "p7", "-tune", "hq", "-profile:v", "main10", "-pix_fmt", "p010le",
|
|
"-rc", "vbr", "-multipass", "fullres", "-spatial-aq", "1", "-temporal-aq", "1", "-rc-lookahead", "32",
|
|
"-bf", "4", "-b_ref_mode", "middle",
|
|
"-b:v", f"{vkbps}k", "-maxrate", f"{vkbps * 2}k", "-bufsize", f"{vkbps * 4}k"]
|
|
for i, a in enumerate(audios):
|
|
ch = int(a.get("channels") or 2)
|
|
layout = {1: "mono", 2: "stereo", 6: "5.1", 8: "7.1"}.get(ch)
|
|
args += ["-map", f"0:{a['index']}", f"-c:a:{i}", "libopus", f"-b:a:{i}", f"{audio_kbps(ch)}k",
|
|
f"-filter:a:{i}", f"aformat=channel_layouts={layout or 'stereo'}"]
|
|
args += ["-mapping_family", "1"]
|
|
for i, s in enumerate(subtitles(info)):
|
|
args += ["-map", f"0:{s['index']}", f"-c:s:{i}", "srt" if s["codec_name"] == "mov_text" else "copy"]
|
|
args += ["-map", "0:t?", "-c:t", "copy", "-metadata", f"COMPACT_TIER={tier}"]
|
|
return args, est
|
|
|
|
|
|
# ── encode, verify, swap ────────────────────────────────────────────────────
|
|
def encode(src, args, tier):
|
|
WORK.mkdir(parents=True, exist_ok=True)
|
|
tmp = WORK / "work.mkv"
|
|
tmp.unlink(missing_ok=True)
|
|
cmd = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-hwaccel", "cuda", "-i", str(src),
|
|
*args, "-metadata", f"COMPACT_SOURCE={src.name} ({src.stat().st_size} bytes)", str(tmp)]
|
|
r = subprocess.run(cmd, capture_output=True, text=True)
|
|
if r.returncode != 0:
|
|
tmp.unlink(missing_ok=True)
|
|
raise RuntimeError("ffmpeg failed: " + r.stderr.strip()[-300:])
|
|
return tmp
|
|
|
|
|
|
def verify(src_info, tmp):
|
|
new = probe(tmp)
|
|
if not new:
|
|
raise RuntimeError("output does not probe")
|
|
a, b = float(src_info["format"]["duration"]), float(new["format"]["duration"])
|
|
if abs(a - b) > max(2.0, a * 0.005):
|
|
raise RuntimeError(f"duration mismatch {a:.1f}s vs {b:.1f}s")
|
|
counts = lambda i: (sum(1 for s in i["streams"] if s["codec_type"] == "audio"), len(subtitles(i)))
|
|
if counts(src_info) != counts(new):
|
|
raise RuntimeError(f"audio/subtitle track counts {counts(src_info)} -> {counts(new)}")
|
|
if int(new["format"]["size"]) >= int(src_info["format"]["size"]):
|
|
raise RuntimeError("output is not smaller")
|
|
return int(new["format"]["size"])
|
|
|
|
|
|
def swap(src, tmp):
|
|
"""Copy the encode next to the original, then rename it over the original in one step."""
|
|
final = src.with_suffix(".mkv")
|
|
staged = src.with_name(f".{final.name}.compact-tmp")
|
|
shutil.copyfile(tmp, staged)
|
|
os.replace(staged, final)
|
|
if final != src:
|
|
src.unlink()
|
|
tmp.unlink(missing_ok=True)
|
|
return final
|
|
|
|
|
|
# ── main ────────────────────────────────────────────────────────────────────
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--until", help="stop starting new encodes after this local time (HH:MM)")
|
|
ap.add_argument("--series", help="only this series title")
|
|
ap.add_argument("--limit", type=int, help="encode at most N files")
|
|
ap.add_argument("--dry-run", action="store_true", help="plan only, change nothing")
|
|
ap.add_argument("--search-all", action="store_true",
|
|
help="search every not-yet-searched title now instead of a few per night")
|
|
opt = ap.parse_args()
|
|
|
|
if (HERE / "PAUSE").exists() and not opt.dry_run:
|
|
log("PAUSE file present, not encoding anything (delete it to resume)")
|
|
return
|
|
|
|
lock = open(HERE / ".lock", "w")
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit("another compact-tv run is active")
|
|
|
|
deadline = None
|
|
if opt.until:
|
|
h, m = map(int, opt.until.split(":"))
|
|
now = datetime.datetime.now()
|
|
deadline = now.replace(hour=h, minute=m, second=0, microsecond=0)
|
|
if deadline <= now:
|
|
deadline += datetime.timedelta(days=1)
|
|
|
|
state = json.loads(STATE.read_text()) if STATE.exists() else {}
|
|
for k in ("failures", "searched", "skip"):
|
|
state.setdefault(k, {})
|
|
save = lambda: STATE.write_text(json.dumps(state, indent=1))
|
|
today = datetime.date.today()
|
|
work = [] # (app, call, title, tier); TV first, then movies
|
|
for app, a in APPS.items():
|
|
call = arr(app)
|
|
profiles = {p["id"]: p["name"] for p in call("/qualityprofile")}
|
|
work += [(app, call, t, profiles[t["qualityProfileId"]]) for t in call(a["titles"])
|
|
if profiles.get(t["qualityProfileId"]) in TIERS and (not opt.series or t["title"] == opt.series)]
|
|
log(f"=== run start{' [dry run]' if opt.dry_run else ''}: {len(work)} title(s) on "
|
|
f"{'/'.join(TIERS)}{f', until {deadline:%H:%M}' if deadline else ''} ===")
|
|
|
|
# ── 1. search: ask the *arr for small replacements before spending GPU ──
|
|
size_of = lambda t: (t.get("statistics") or {}).get("sizeOnDisk") or t.get("sizeOnDisk") or 0
|
|
for app in APPS:
|
|
fresh = sorted((w for w in work if w[0] == app and f"{app}:{w[2]['id']}" not in state["searched"]
|
|
and size_of(w[2]) > 0), key=lambda w: -size_of(w[2]))
|
|
fresh = fresh if opt.search_all else fresh[:SEARCHES_PER_NIGHT[app]]
|
|
for _, call, t, tier in fresh:
|
|
log(f" {'would search' if opt.dry_run else 'search'} {t['title']} ({size_of(t) / 1e9:.0f} GB, {tier})")
|
|
if fresh and not opt.dry_run:
|
|
call = fresh[0][1]
|
|
if app == "sonarr":
|
|
for _, _, t, _ in fresh:
|
|
call("/command", {"name": "SeriesSearch", "seriesId": t["id"]})
|
|
else:
|
|
call("/command", {"name": "MoviesSearch", "movieIds": [t["id"] for _, _, t, _ in fresh]})
|
|
for _, _, t, _ in fresh:
|
|
state["searched"][f"{app}:{t['id']}"] = today.isoformat()
|
|
save()
|
|
|
|
# ── 2. encode what is still big after its search ────────────────────────
|
|
age = lambda iso: (today - datetime.date.fromisoformat(iso)).days
|
|
|
|
def ready(app, ef, title_searched, to_search):
|
|
"""True if this big file has had its chance to be replaced by a small download."""
|
|
if (ef.get("dateAdded") or "")[:10] <= title_searched: # was on disk when the title was searched
|
|
return age(title_searched) >= SEARCH_GRACE_DAYS
|
|
key = f"{app}:file:{ef['id']}" # arrived later: search it on its own first
|
|
if key not in state["searched"]:
|
|
to_search.append(ef)
|
|
return False
|
|
return age(state["searched"][key]) >= FILE_SEARCH_GRACE_DAYS
|
|
|
|
def search_files(app, call, show, files):
|
|
if opt.dry_run:
|
|
log(f" would search {len(files)} newer file(s) of {show['title']}"); return
|
|
if app == "sonarr":
|
|
ids = {f["id"] for f in files}
|
|
eps = [e["id"] for e in call(f"/episode?seriesId={show['id']}") if e.get("episodeFileId") in ids]
|
|
call("/command", {"name": "EpisodeSearch", "episodeIds": eps})
|
|
else:
|
|
call("/command", {"name": "MoviesSearch", "movieIds": [show["id"]]})
|
|
for f in files:
|
|
state["searched"][f"{app}:file:{f['id']}"] = today.isoformat()
|
|
log(f" search {len(files)} newer file(s) of {show['title']} before encoding"); save()
|
|
|
|
done = saved = 0
|
|
for app, call, show, tier in work:
|
|
title_searched = state["searched"].get(f"{app}:{show['id']}")
|
|
if not title_searched:
|
|
continue # its title search comes first, on a later night
|
|
container, host = APPS[app]["paths"]
|
|
files = sorted(APPS[app]["files"](call, show), key=lambda f: f["path"])
|
|
changed, to_search = False, []
|
|
for ef in files:
|
|
if deadline and datetime.datetime.now() >= deadline:
|
|
log("deadline reached, stopping"); break
|
|
if opt.limit is not None and done >= opt.limit:
|
|
break
|
|
src = pathlib.Path(ef["path"].replace(container, host, 1))
|
|
if state["failures"].get(str(src), 0) >= MAX_FAILURES or not src.exists():
|
|
continue
|
|
# Files already ruled out at this tier and size need no re-probe (most of them, most nights).
|
|
seen = f"{tier}:{src.stat().st_size}"
|
|
if state["skip"].get(str(src)) == seen:
|
|
continue
|
|
info = probe(src)
|
|
if not info:
|
|
log(f" SKIP {src.name}: does not probe"); continue
|
|
args, est = plan(info, tier, app)
|
|
if args is None:
|
|
state["skip"][str(src)] = seen # reason in `est`; quiet, since most nights most files are done
|
|
continue
|
|
if not ready(app, ef, title_searched, to_search):
|
|
continue
|
|
old = int(info["format"]["size"])
|
|
if opt.dry_run:
|
|
log(f" would encode {src.name}: {old / 1e6:.0f} MB -> ~{est / 1e6:.0f} MB ({tier})")
|
|
done += 1; continue
|
|
t0 = time.monotonic()
|
|
try:
|
|
tmp = encode(src, args, tier)
|
|
new = verify(info, tmp)
|
|
final = swap(src, tmp)
|
|
except Exception as e:
|
|
state["failures"][str(src)] = state["failures"].get(str(src), 0) + 1
|
|
save()
|
|
log(f" FAIL {src.name}: {e}"); continue
|
|
done += 1; saved += old - new; changed = True
|
|
log(f" {final.name}: {old / 1e6:.0f} MB -> {new / 1e6:.0f} MB "
|
|
f"(-{(1 - new / old) * 100:.0f}%, {time.monotonic() - t0:.0f}s, {tier})")
|
|
if to_search:
|
|
search_files(app, call, show, to_search)
|
|
if changed:
|
|
call("/command", APPS[app]["rescan"](show))
|
|
if (deadline and datetime.datetime.now() >= deadline) or (opt.limit is not None and done >= opt.limit):
|
|
break
|
|
if not opt.dry_run:
|
|
save()
|
|
log(f"=== run end: {done} file(s){' planned' if opt.dry_run else ''}, saved {saved / 1e9:.1f} GB ===")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|