github-to-gitea-mirror/sync_mirrors.py

#!/usr/bin/env python3
"""Mirror all GitHub repositories of a user into Gitea as pull mirrors.

Existing mirrors are left to Gitea's own cron; this script only creates the
ones that are missing and keeps the mirror interval aligned with the config.
It never deletes anything on the Gitea side -- a mirror that vanished upstream
is exactly what a backup is supposed to keep.
"""

import json
import os
import re
import sys
import urllib.error
import urllib.request
from datetime import datetime, timezone

KNOWN_KEYS = (
    "GITHUB_USER",
    "GITHUB_TOKEN",
    "GITEA_URL",
    "GITEA_TOKEN",
    "GITEA_OWNER",
    "MIRROR_INTERVAL",
    "INCLUDE_FORKS",
    "INCLUDE_ARCHIVED",
    "DRY_RUN",
)

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_PATH = os.environ.get("MIRROR_ENV", os.path.join(SCRIPT_DIR, "mirror.env"))

errors = 0
last_headers = {}


def log(level, msg):
    print("%s [%s] %s" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), level, msg), flush=True)


def fail(msg):
    global errors
    errors += 1
    log("ERROR", msg)


def load_config():
    cfg = {
        "GITHUB_USER": "",
        "GITHUB_TOKEN": "",
        "GITEA_URL": "http://127.0.0.1:3000",
        "GITEA_TOKEN": "",
        "GITEA_OWNER": "",
        "MIRROR_INTERVAL": "24h",
        "INCLUDE_FORKS": "false",
        "INCLUDE_ARCHIVED": "true",
        "DRY_RUN": "false",
    }
    if os.path.exists(CONFIG_PATH):
        with open(CONFIG_PATH, encoding="utf-8") as fh:
            for line in fh:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, value = line.partition("=")
                key = key.strip()
                if key in KNOWN_KEYS:
                    cfg[key] = value.strip().strip('"').strip("'")
    else:
        log("WARN", "No config file at %s, relying on environment" % CONFIG_PATH)

    for key in KNOWN_KEYS:
        if os.environ.get(key):
            cfg[key] = os.environ[key]

    cfg["GITEA_URL"] = cfg["GITEA_URL"].rstrip("/")
    return cfg


def truthy(value):
    return str(value).strip().lower() in ("1", "true", "yes", "on")


def api(url, token=None, scheme="token", method="GET", payload=None):
    """Return (status, parsed_body). Raises urllib.error.HTTPError on >=400."""
    data = None
    headers = {
        "Accept": "application/json",
        "User-Agent": "gitea-github-mirror/1.0",
    }
    if payload is not None:
        data = json.dumps(payload).encode("utf-8")
        headers["Content-Type"] = "application/json"
    if token:
        headers["Authorization"] = "%s %s" % (scheme, token)

    req = urllib.request.Request(url, data=data, headers=headers, method=method)
    with urllib.request.urlopen(req, timeout=180) as resp:
        body = resp.read().decode("utf-8")
        last_headers.clear()
        last_headers.update({k.lower(): v for k, v in resp.headers.items()})
        return resp.status, (json.loads(body) if body.strip() else None)


def github_repos(cfg):
    """All repos owned by the user; private ones included when a token is set."""
    token = cfg["GITHUB_TOKEN"]
    repos = []
    page = 1
    while True:
        if token:
            url = ("https://api.github.com/user/repos"
                   "?per_page=100&affiliation=owner&page=%d" % page)
        else:
            url = ("https://api.github.com/users/%s/repos"
                   "?per_page=100&type=owner&page=%d" % (cfg["GITHUB_USER"], page))
        _, batch = api(url, token=token, scheme="Bearer")
        if not batch:
            break
        repos.extend(batch)
        if len(batch) < 100:
            break
        page += 1

    if token:
        # affiliation=owner still returns repos owned by orgs the user belongs to
        repos = [r for r in repos
                 if r["owner"]["login"].lower() == cfg["GITHUB_USER"].lower()]
    return repos


def warn_token_expiry():
    """Fine-grained PATs always expire; a silent expiry would stall the mirror.

    GitHub reports the date in a response header, so every run can check it.
    """
    raw = last_headers.get("github-authentication-token-expiration")
    if not raw:
        return  # classic token without expiry, or no token at all
    text = raw.strip().replace(" UTC", "").replace("Z", "").replace("T", " ")
    parsed = None
    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d"):
        try:
            parsed = datetime.strptime(text.split(".")[0], fmt)
            break
        except ValueError:
            continue
    if parsed is None:
        return

    now_utc = datetime.now(timezone.utc).replace(tzinfo=None)
    days = (parsed.replace(tzinfo=None) - now_utc).days
    if days < 0:
        fail("GITHUB_TOKEN expired on %s - renew it" % parsed.date())
    elif days <= 14:
        log("WARN", "GITHUB_TOKEN expires in %d day(s), on %s - renew it"
            % (days, parsed.date()))
    else:
        log("INFO", "GITHUB_TOKEN valid until %s (%d days)" % (parsed.date(), days))


def duration_seconds(value):
    """Parse a Go-style duration ('24h', '8h0m0s') into seconds. -1 if unparseable."""
    if not value:
        return -1
    total = 0
    matched = False
    for amount, unit in re.findall(r"(\d+)\s*([hms])", str(value)):
        matched = True
        total += int(amount) * {"h": 3600, "m": 60, "s": 1}[unit]
    return total if matched else -1


def gitea_repo(cfg, name):
    try:
        _, repo = api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
                      token=cfg["GITEA_TOKEN"])
        return repo
    except urllib.error.HTTPError as exc:
        if exc.code == 404:
            return None
        raise


def create_mirror(cfg, repo, dry_run):
    name = repo["name"]
    payload = {
        "clone_addr": repo["clone_url"],
        "repo_name": name,
        "repo_owner": cfg["GITEA_OWNER"],
        "mirror": True,
        "mirror_interval": cfg["MIRROR_INTERVAL"],
        "private": bool(repo.get("private")),
        "description": (repo.get("description") or "")[:255],
        "service": "github",
        # Pull mirrors only ever sync git data, so importing the rest would be a
        # one-off snapshot that silently goes stale. Keep it lean on the Pi.
        "wiki": False,
        "issues": False,
        "labels": False,
        "milestones": False,
        "releases": False,
        "pull_requests": False,
    }
    if cfg["GITHUB_TOKEN"]:
        payload["auth_token"] = cfg["GITHUB_TOKEN"]

    if dry_run:
        log("DRY", "would mirror %s (private=%s)" % (name, payload["private"]))
        return

    api("%s/api/v1/repos/migrate" % cfg["GITEA_URL"], token=cfg["GITEA_TOKEN"],
        method="POST", payload=payload)
    log("OK", "mirrored %s (private=%s)" % (name, payload["private"]))


def align_interval(cfg, name, existing, dry_run):
    wanted = duration_seconds(cfg["MIRROR_INTERVAL"])
    current = duration_seconds(existing.get("mirror_interval"))
    if wanted < 0 or current < 0 or wanted == current:
        return
    if dry_run:
        log("DRY", "would set %s interval %s -> %s"
            % (name, existing.get("mirror_interval"), cfg["MIRROR_INTERVAL"]))
        return
    api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
        token=cfg["GITEA_TOKEN"], method="PATCH",
        payload={"mirror_interval": cfg["MIRROR_INTERVAL"]})
    log("OK", "interval of %s set to %s" % (name, cfg["MIRROR_INTERVAL"]))


def main():
    cfg = load_config()
    dry_run = truthy(cfg["DRY_RUN"])

    missing = [k for k in ("GITHUB_USER", "GITEA_URL", "GITEA_TOKEN", "GITEA_OWNER")
               if not cfg[k]]
    if missing:
        log("ERROR", "Missing config value(s): %s" % ", ".join(missing))
        return 2

    if not cfg["GITHUB_TOKEN"]:
        log("WARN", "GITHUB_TOKEN is empty - private repositories will be skipped")

    try:
        repos = github_repos(cfg)
    except urllib.error.HTTPError as exc:
        log("ERROR", "GitHub API %s: %s" % (exc.code, exc.read().decode("utf-8", "replace")[:300]))
        return 2
    except Exception as exc:
        log("ERROR", "GitHub API unreachable: %s" % exc)
        return 2

    warn_token_expiry()  # must run before any Gitea call overwrites the headers

    if not truthy(cfg["INCLUDE_FORKS"]):
        repos = [r for r in repos if not r.get("fork")]
    if not truthy(cfg["INCLUDE_ARCHIVED"]):
        repos = [r for r in repos if not r.get("archived")]

    private_count = sum(1 for r in repos if r.get("private"))
    log("INFO", "GitHub: %d repositor%s to consider (%d public, %d private)%s"
        % (len(repos), "y" if len(repos) == 1 else "ies",
           len(repos) - private_count, private_count,
           " (DRY RUN)" if dry_run else ""))

    # A fine-grained token defaults to "Public repositories", which looks exactly
    # like an account without private repos. Name the likely cause explicitly.
    if cfg["GITHUB_TOKEN"] and private_count == 0:
        log("WARN", "Token is set but GitHub reports no private repositories - "
                    "check that its repository access is 'All repositories', "
                    "not 'Public repositories'")

    created = skipped = 0
    for repo in sorted(repos, key=lambda r: r["name"].lower()):
        name = repo["name"]
        try:
            existing = gitea_repo(cfg, name)
        except Exception as exc:
            fail("%s: cannot query Gitea: %s" % (name, exc))
            continue

        if existing is None:
            try:
                create_mirror(cfg, repo, dry_run)
                created += 1
            except urllib.error.HTTPError as exc:
                detail = exc.read().decode("utf-8", "replace")[:300]
                fail("%s: migrate failed (HTTP %s): %s" % (name, exc.code, detail))
            except Exception as exc:
                fail("%s: migrate failed: %s" % (name, exc))
            continue

        skipped += 1
        if not existing.get("mirror"):
            log("WARN", "%s exists in Gitea but is not a mirror - left untouched" % name)
            continue
        try:
            align_interval(cfg, name, existing, dry_run)
        except Exception as exc:
            fail("%s: could not update mirror interval: %s" % (name, exc))

    github_names = {r["name"].lower() for r in repos}
    try:
        page, orphans = 1, []
        while True:
            _, batch = api("%s/api/v1/user/repos?limit=50&page=%d" % (cfg["GITEA_URL"], page),
                           token=cfg["GITEA_TOKEN"])
            if not batch:
                break
            orphans.extend(
                r["name"] for r in batch
                if r.get("mirror")
                and r["owner"]["login"].lower() == cfg["GITEA_OWNER"].lower()
                and r["name"].lower() not in github_names
            )
            if len(batch) < 50:
                break
            page += 1
        if orphans:
            log("WARN", "Mirrors with no GitHub counterpart (renamed or deleted upstream): %s"
                % ", ".join(sorted(orphans)))
    except Exception as exc:
        log("WARN", "Could not check for orphaned mirrors: %s" % exc)

    log("INFO", "Done: %d created, %d already present, %d error(s)" % (created, skipped, errors))
    return 1 if errors else 0


if __name__ == "__main__":
    sys.exit(main())