takana/scripts/fuentes/mirror-git-poblar.py

#!/usr/bin/env python3
"""Genera y sube bundles shallow de las fuentes git del corpus. Ver ADR 0013 y el .sh hermano.

Es INCREMENTAL: consulta primero qué commits ya están en el mirror y sólo trabaja los que faltan.
Interrumpible sin penalización — cada bundle es independiente.
"""
import os, re, subprocess, sys, glob, tempfile, shutil

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
os.chdir(ROOT)
SB_USER = os.environ.get("SB_USER", "u647150")
SB_HOST = os.environ.get("SB_HOST", "u647150.your-storagebox.de")
SB_PORT = os.environ.get("SB_PORT", "23")
KEY     = os.environ.get("KEY", os.path.expanduser("~/.ssh/github5"))
LIMITE  = int(os.environ.get("LIMITE", "0"))
SOLO    = os.environ.get("SOLO", "")
DEST    = "hammer/fuentes-git"
SSH = ["ssh", "-4", "-p", SB_PORT, "-i", KEY, "-o", "StrictHostKeyChecking=accept-new",
       "-o", "ConnectTimeout=30"]

def fuentes_git():
    vistos, out = set(), []
    for f in sorted(glob.glob("recipes/**/*.toml", recursive=True)):
        s = open(f, errors="ignore").read()
        repo = re.search(r'^\s*repo\s*=\s*"([^"]+)"', s, re.M)
        com  = re.search(r'^\s*commit\s*=\s*"([0-9a-fA-F]{7,40})"', s, re.M)
        if not (repo and com):
            continue
        nombre = (re.search(r'^\s*name\s*=\s*"([^"]+)"', s, re.M) or [None, os.path.basename(f)[:-5]])[1]
        c = com.group(1).lower()
        if c in vistos:              # el mismo commit desde dos colas = un solo bundle
            continue
        vistos.add(c)
        out.append({"receta": nombre, "repo": repo.group(1), "commit": c})
    return out

def ya_en_mirror():
    r = subprocess.run(SSH + [f"{SB_USER}@{SB_HOST}", f"ls {DEST}"],
                       capture_output=True, text=True, timeout=180)
    return {l.strip()[:-7] for l in r.stdout.splitlines() if l.strip().endswith(".bundle")}

def bundle(e, tmpdir):
    """Clona --depth 1 el commit exacto y emite un bundle. None si upstream no lo da."""
    d = os.path.join(tmpdir, "r")
    shutil.rmtree(d, ignore_errors=True)
    os.makedirs(d)
    def git(*a, **kw):
        return subprocess.run(["git", "-C", d] + list(a), capture_output=True, text=True,
                              timeout=kw.get("t", 900))
    if git("init", "-q").returncode != 0: return None
    if git("remote", "add", "origin", e["repo"]).returncode != 0: return None
    # --depth 1 del SHA exacto: el servidor debe permitir fetch por sha (uploadpack.allowReachableSHA1).
    if git("fetch", "-q", "--depth", "1", "origin", e["commit"], t=1800).returncode != 0:
        return None
    if git("update-ref", "refs/heads/hammer", e["commit"]).returncode != 0: return None
    out = os.path.join(tmpdir, f'{e["commit"]}.bundle')
    if git("bundle", "create", out, "refs/heads/hammer", t=1800).returncode != 0: return None
    return out if os.path.exists(out) else None

def main():
    fs = fuentes_git()
    if SOLO: fs = [e for e in fs if e["receta"] == SOLO]
    print(f"== fuentes git en el corpus: {len(fs)} commits distintos", flush=True)
    try:
        hay = ya_en_mirror()
    except Exception as ex:
        print(f"!! no pude listar el mirror ({ex}); asumo vacío", flush=True); hay = set()
    print(f"== ya en el mirror: {len(hay)}", flush=True)
    faltan = [e for e in fs if e["commit"] not in hay]
    if LIMITE: faltan = faltan[:LIMITE]
    print(f"== a espejar ahora: {len(faltan)}", flush=True)

    subprocess.run(SSH + [f"{SB_USER}@{SB_HOST}", f"mkdir hammer {DEST}"],
                   capture_output=True, text=True)
    ok = fallo = 0
    tmpdir = tempfile.mkdtemp(prefix="hammer-git-mirror-")
    try:
        for i, e in enumerate(faltan, 1):
            b = bundle(e, tmpdir)
            if not b:
                fallo += 1
                print(f"  [{i}/{len(faltan)}] {e['receta']:<24} ✗ upstream no da el commit", flush=True)
                continue
            mb = os.path.getsize(b) / 1e6
            r = subprocess.run(["rsync", "-a", "-e", " ".join(SSH), b,
                                f"{SB_USER}@{SB_HOST}:{DEST}/"], capture_output=True, text=True, timeout=3600)
            if r.returncode == 0:
                ok += 1
                print(f"  [{i}/{len(faltan)}] {e['receta']:<24} ✓ {mb:6.1f} MB", flush=True)
            else:
                fallo += 1
                print(f"  [{i}/{len(faltan)}] {e['receta']:<24} ✗ subida: {r.stderr.strip()[:60]}", flush=True)
            os.remove(b)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True)
    print(f"== espejados {ok}   fallidos {fallo}")
    return 0

sys.exit(main())