commit d7f5097d1f692ffbc85419dec26678040b6a739a Author: Andre <200930841+Andre-Bzk@users.noreply.github.com> Date: Sat Aug 8 08:48:07 2026 +0200 git init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..44b49ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Real configuration -- contains GitHub and Gitea tokens. +mirror.env + +# Runtime output +sync.log +sync.log.tmp + +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..0e274a0 --- /dev/null +++ b/README.md @@ -0,0 +1,321 @@ +# GitHub → Gitea Mirror + +[English](#english) | [Deutsch](#deutsch) + +--- + +## English + +Mirrors **all** GitHub repositories of a user into a self-hosted Gitea instance +as pull mirrors — including repositories created later. + +Gitea can already mirror a single repository through its web UI. What it cannot +do is watch an account and pick up new repositories on its own. That gap is the +only reason this script exists: a daily cron job asks the GitHub API which +repositories exist and creates the mirrors that are missing. Keeping their +*content* up to date remains Gitea's job. + +| Task | Handled by | Frequency | +| --- | --- | --- | +| Discover **new** repositories, create mirrors | this script (cron) | daily | +| Pull new commits into **existing** mirrors | Gitea itself | `MIRROR_INTERVAL` | + +### Requirements + +- Gitea 1.20+ with an API token (scopes `write:repository,read:user`) +- Python 3.7+ — standard library only, no `pip install`, no `jq` +- `cron` + +Deliberately dependency-free so it runs on a small box: on a Raspberry Pi 3B +with 1 GB RAM the sync is invisible in `free -h`. + +### Installation + +```bash +git clone gitea-github-mirror +cd gitea-github-mirror +./install.sh +``` + +`install.sh` copies the scripts to `~/gitea-github-mirror`, creates `mirror.env` +from the example, fixes permissions and adds the cron entry. It is safe to +re-run — an existing `mirror.env` is never overwritten. + +```bash +TARGET=/opt/gitea-mirror ./install.sh # different directory +CRON_TIME="0 3" ./install.sh # different schedule +``` + +Create the Gitea token on the Gitea host: + +```bash +docker exec -u git gitea admin user generate-access-token \ + --username --token-name github-mirror-sync \ + --scopes write:repository,read:user +``` + +Then fill in `mirror.env` and verify: + +```bash +cd ~/gitea-github-mirror +DRY_RUN=true python3 sync_mirrors.py # preview, changes nothing +python3 sync_mirrors.py # run for real +``` + +### Configuration (`mirror.env`) + +| Key | Meaning | +| --- | --- | +| `GITHUB_USER` | GitHub account to mirror | +| `GITHUB_TOKEN` | See below. Empty = public repositories only | +| `GITEA_URL` | Gitea base URL as seen from this machine | +| `GITEA_TOKEN` | Gitea API token | +| `GITEA_OWNER` | Gitea user the mirrors belong to | +| `MIRROR_INTERVAL` | Go duration, e.g. `24h`, `8h`, `30m` | +| `INCLUDE_FORKS` | Mirror forks too (default `false`) | +| `INCLUDE_ARCHIVED` | Mirror archived repositories (default `true`) | +| `DRY_RUN` | `true` = show what would happen, change nothing | + +Every key can be overridden by an environment variable of the same name, which +is what `DRY_RUN=true python3 sync_mirrors.py` uses. + +### GitHub token + +Needed only for private repositories. Use a **fine-grained** token +(): + +- Repository access: **All repositories** — required, otherwise repositories + created later are invisible to the script and auto-discovery is pointless. +- Permissions: **Contents: Read-only**. Metadata read is added automatically, + everything else stays on "No access". +- Expiration: **No expiration** is the sensible choice for an unattended cron + job — an expiring token stalls the mirror without anyone noticing. The token + can only read, so an unlimited lifetime costs little here. + +A classic token with scope `repo` works too, but grants full write and delete +access to every repository — far more than a read-only mirror needs, for a +token that sits in plaintext on the server. + +If you do set an expiry (or your organization enforces one), each run logs the +remaining validity, warns from 14 days out and exits non-zero once expired: + +``` +[INFO] GITHUB_TOKEN valid until 2026-12-31 (145 days) +[WARN] GITHUB_TOKEN expires in 7 day(s), on 2026-08-15 - renew it +[ERROR] GITHUB_TOKEN expired on 2026-08-01 - renew it +``` + +### Scope and limits + +- Mirrored is **git data only**: all branches and tags. Issues, pull requests, + releases and the wiki are excluded on purpose — a pull mirror never syncs + those, so importing them once would leave a snapshot that silently goes + stale. To change that, flip the flags in `create_mirror()`. +- The script **never deletes** anything in Gitea. If a repository disappears or + is renamed on GitHub, its mirror stays and is reported as an orphan in the + log — for a backup that is the right direction. +- A Gitea repository that exists but is **not** a mirror is left untouched, so + a real working repository can never be overwritten. +- Mirrors are read-only in Gitea. Push to GitHub, not to the mirror. + +### Files + +| File | Purpose | +| --- | --- | +| `sync_mirrors.py` | Creates missing mirrors, aligns the mirror interval | +| `run_sync.sh` | Cron wrapper: appends to `sync.log`, trims it to 1000 lines | +| `install.sh` | Installs scripts, config and cron entry | +| `mirror.env.example` | Configuration template | +| `mirror.env` | Real config with secrets — git-ignored, `chmod 600` | +| `sync.log` | Last ~2000 log lines | + +### Troubleshooting + +```bash +tail -f ~/gitea-github-mirror/sync.log +``` + +| Log line | Cause | +| --- | --- | +| `GitHub API 401` | Token invalid or expired | +| `GitHub API 403` | Rate limit, or token lacks repository access | +| `migrate failed (HTTP 409)` | Name already taken in Gitea | +| `... exists in Gitea but is not a mirror` | Real repo with that name — rename one of them | +| `Missing config value(s)` | `mirror.env` not filled in | +| `no private repositories` despite a token | Token's repository access is "Public repositories", or no repository permission is set at all | + +**Fine-grained token shows no private repositories.** Repository access and +permissions are two separate settings and both are needed: access decides +*which* repositories the token covers, permissions decide *what* it may do with +them. With "All repositories" but an empty permission list the token can see +nothing, and private repositories do not even appear in the listing. Set +**Contents: Read-only**; Metadata read is added automatically. + +**After rotating the GitHub token**, updating `mirror.env` is not enough. Gitea +stores the credentials inside each private mirror's remote URL when the mirror +is created, so existing private mirrors keep using the old token and start +failing silently. Either update *Settings → Mirror Settings* per repository in +Gitea, or delete the private mirrors and let the next run recreate them. Public +mirrors are unaffected — they clone without credentials. + +--- + +## Deutsch + +Spiegelt **alle** GitHub-Repositories eines Benutzers als Pull-Mirror in eine +selbst gehostete Gitea-Instanz — auch Repositories, die erst später entstehen. + +Ein einzelnes Repository kann Gitea bereits über die Weboberfläche spiegeln. +Was Gitea nicht kann: einen Account beobachten und neue Repositories von selbst +aufnehmen. Genau dafür gibt es dieses Skript — ein täglicher Cron-Job fragt die +GitHub-API, welche Repositories existieren, und legt die fehlenden Mirrors an. +Deren *Inhalt* aktuell zu halten bleibt Giteas Aufgabe. + +| Aufgabe | Erledigt von | Häufigkeit | +| --- | --- | --- | +| **Neue** Repositories finden, Mirror anlegen | dieses Skript (Cron) | täglich | +| Commits in **bestehende** Mirrors holen | Gitea selbst | `MIRROR_INTERVAL` | + +### Voraussetzungen + +- Gitea 1.20+ mit API-Token (Scopes `write:repository,read:user`) +- Python 3.7+ — nur Standardbibliothek, kein `pip install`, kein `jq` +- `cron` + +Bewusst ohne Abhängigkeiten, damit es auf kleiner Hardware läuft: Auf einem +Raspberry Pi 3B mit 1 GB RAM ist der Sync in `free -h` nicht zu sehen. + +### Installation + +```bash +git clone gitea-github-mirror +cd gitea-github-mirror +./install.sh +``` + +`install.sh` kopiert die Skripte nach `~/gitea-github-mirror`, legt `mirror.env` +aus der Vorlage an, setzt die Rechte und trägt den Cron-Job ein. Mehrfaches +Ausführen ist unbedenklich — eine vorhandene `mirror.env` wird nie überschrieben. + +```bash +TARGET=/opt/gitea-mirror ./install.sh # anderes Verzeichnis +CRON_TIME="0 3" ./install.sh # andere Uhrzeit +``` + +Den Gitea-Token auf dem Gitea-Host erzeugen: + +```bash +docker exec -u git gitea admin user generate-access-token \ + --username --token-name github-mirror-sync \ + --scopes write:repository,read:user +``` + +Danach `mirror.env` ausfüllen und prüfen: + +```bash +cd ~/gitea-github-mirror +DRY_RUN=true python3 sync_mirrors.py # Vorschau, ändert nichts +python3 sync_mirrors.py # echter Lauf +``` + +### Konfiguration (`mirror.env`) + +| Schlüssel | Bedeutung | +| --- | --- | +| `GITHUB_USER` | Zu spiegelnder GitHub-Account | +| `GITHUB_TOKEN` | Siehe unten. Leer = nur öffentliche Repositories | +| `GITEA_URL` | Gitea-Basis-URL, von dieser Maschine aus gesehen | +| `GITEA_TOKEN` | Gitea-API-Token | +| `GITEA_OWNER` | Gitea-Benutzer, dem die Mirrors gehören | +| `MIRROR_INTERVAL` | Go-Duration, z. B. `24h`, `8h`, `30m` | +| `INCLUDE_FORKS` | Forks mitspiegeln (Standard `false`) | +| `INCLUDE_ARCHIVED` | Archivierte Repositories spiegeln (Standard `true`) | +| `DRY_RUN` | `true` = nur anzeigen, nichts ändern | + +Jeder Schlüssel lässt sich durch eine gleichnamige Umgebungsvariable +überschreiben — genau das nutzt `DRY_RUN=true python3 sync_mirrors.py`. + +### GitHub-Token + +Nur für private Repositories nötig. Empfohlen ist ein **fine-grained** Token +(): + +- Repository access: **All repositories** — zwingend, sonst sind später + angelegte Repositories für das Skript unsichtbar und die automatische + Erkennung wäre sinnlos. +- Permissions: **Contents: Read-only**. Metadata-Lesen kommt automatisch dazu, + alles andere bleibt auf „No access". +- Expiration: **No expiration** ist für einen unbeaufsichtigten Cron-Job die + sinnvolle Wahl — ein ablaufender Token legt den Spiegel still, ohne dass es + jemand merkt. Der Token kann ohnehin nur lesen, unbegrenzte Laufzeit kostet + hier also wenig. + +Ein Classic-Token mit Scope `repo` funktioniert ebenfalls, gibt aber Vollzugriff +zum Schreiben und Löschen auf jedes Repository — deutlich mehr, als ein +Lese-Spiegel braucht, für einen Token, der im Klartext auf dem Server liegt. + +Wird doch ein Ablaufdatum gesetzt (oder von der Organisation erzwungen), +protokolliert jeder Lauf die Restlaufzeit, warnt ab 14 Tagen und endet nach +Ablauf mit Exit-Code 1: + +``` +[INFO] GITHUB_TOKEN valid until 2026-12-31 (145 days) +[WARN] GITHUB_TOKEN expires in 7 day(s), on 2026-08-15 - renew it +[ERROR] GITHUB_TOKEN expired on 2026-08-01 - renew it +``` + +### Umfang und Grenzen + +- Gespiegelt werden **nur Git-Daten**: alle Branches und Tags. Issues, Pull + Requests, Releases und Wiki bleiben bewusst außen vor — ein Pull-Mirror + synchronisiert das nie, ein einmaliger Import würde also stillschweigend + veralten. Umstellbar über die Flags in `create_mirror()`. +- Das Skript **löscht nie** etwas in Gitea. Verschwindet ein Repository auf + GitHub oder wird umbenannt, bleibt der Mirror bestehen und wird im Log als + verwaist gemeldet — für ein Backup ist das die richtige Richtung. +- Ein Gitea-Repository, das existiert, aber **kein** Mirror ist, wird nicht + angefasst; ein echtes Arbeits-Repository kann so nie überschrieben werden. +- Mirrors sind in Gitea schreibgeschützt. Gepusht wird nach GitHub, nicht in + den Spiegel. + +### Dateien + +| Datei | Zweck | +| --- | --- | +| `sync_mirrors.py` | Legt fehlende Mirrors an, gleicht das Mirror-Intervall ab | +| `run_sync.sh` | Cron-Wrapper: schreibt nach `sync.log`, kürzt es auf 1000 Zeilen | +| `install.sh` | Installiert Skripte, Konfiguration und Cron-Eintrag | +| `mirror.env.example` | Konfigurationsvorlage | +| `mirror.env` | Echte Konfiguration mit Secrets — git-ignoriert, `chmod 600` | +| `sync.log` | Die letzten ~2000 Log-Zeilen | + +### Fehlersuche + +```bash +tail -f ~/gitea-github-mirror/sync.log +``` + +| Log-Zeile | Ursache | +| --- | --- | +| `GitHub API 401` | Token ungültig oder abgelaufen | +| `GitHub API 403` | Rate Limit, oder Token hat keinen Repository-Zugriff | +| `migrate failed (HTTP 409)` | Name in Gitea bereits vergeben | +| `... exists in Gitea but is not a mirror` | Echtes Repo gleichen Namens — eines umbenennen | +| `Missing config value(s)` | `mirror.env` nicht ausgefüllt | +| `no private repositories` trotz Token | Repository access steht auf „Public repositories", oder es ist gar keine Repository-Permission gesetzt | + +**Fine-grained Token zeigt keine privaten Repos.** Repository access und +Permissions sind zwei getrennte Einstellungen, beide werden gebraucht: Der +Access bestimmt, *welche* Repositories der Token betrifft, die Permissions, +*was* er damit darf. Mit „All repositories", aber leerer Permission-Liste sieht +der Token nichts — private Repositories erscheinen dann nicht einmal in der +Auflistung. **Contents: Read-only** setzen, Metadata-Lesen kommt automatisch +dazu. + +**Nach dem Wechsel des GitHub-Tokens** genügt es nicht, `mirror.env` +anzupassen. Gitea hinterlegt die Zugangsdaten beim Anlegen eines privaten +Mirrors fest in dessen Remote-URL — bestehende private Mirrors arbeiten also +weiter mit dem alten Token und scheitern still. Entweder pro Repository unter +*Einstellungen → Mirror-Einstellungen* in Gitea nachziehen, oder die privaten +Mirrors löschen und vom nächsten Lauf neu anlegen lassen. Öffentliche Mirrors +sind nicht betroffen, sie klonen ohne Zugangsdaten. diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..491241c --- /dev/null +++ b/install.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Install the GitHub -> Gitea mirror sync on this machine. +# +# Safe to re-run: an existing mirror.env is never overwritten and the cron +# entry is only added once. +# +# TARGET=/opt/gitea-mirror ./install.sh # override install directory +# CRON_TIME="0 3" ./install.sh # override schedule (minute hour) +set -eu + +SRC="$(dirname "$(readlink -f "$0")")" +TARGET="${TARGET:-$HOME/gitea-github-mirror}" +CRON_TIME="${CRON_TIME:-30 4}" + +say() { printf '%s\n' "$*"; } +die() { printf 'error: %s\n' "$*" >&2; exit 1; } + +# --- prerequisites --- +command -v python3 >/dev/null 2>&1 || die "python3 not found" +python3 -c 'import json, urllib.request' 2>/dev/null \ + || die "python3 is missing the standard library modules json/urllib" +command -v crontab >/dev/null 2>&1 || die "crontab not found (install cron)" +say "python3 $(python3 -c 'import platform; print(platform.python_version())') ok" + +# --- files --- +mkdir -p "$TARGET" +for f in sync_mirrors.py run_sync.sh README.md; do + [ -f "$SRC/$f" ] || die "missing source file: $f" + cp "$SRC/$f" "$TARGET/$f" + # Files may arrive from Windows; CRLF would break the shebang. + sed -i 's/\r$//' "$TARGET/$f" +done +chmod 700 "$TARGET/sync_mirrors.py" "$TARGET/run_sync.sh" +say "installed scripts in $TARGET" + +if [ -f "$TARGET/mirror.env" ]; then + chmod 600 "$TARGET/mirror.env" + say "kept existing mirror.env" + CONFIGURED=1 +else + cp "$SRC/mirror.env.example" "$TARGET/mirror.env" + sed -i 's/\r$//' "$TARGET/mirror.env" + chmod 600 "$TARGET/mirror.env" + say "created $TARGET/mirror.env from the example" + CONFIGURED=0 +fi + +# --- cron --- +CRON_LINE="$CRON_TIME * * * $TARGET/run_sync.sh" +if crontab -l 2>/dev/null | grep -Fq "$TARGET/run_sync.sh"; then + say "cron entry already present" +else + { crontab -l 2>/dev/null || true; printf '%s\n' "$CRON_LINE"; } | crontab - + say "cron entry added: $CRON_LINE" +fi + +say "" +if [ "$CONFIGURED" = "0" ]; then + say "Next: edit $TARGET/mirror.env (GITHUB_USER, GITEA_TOKEN, GITEA_OWNER)," + say "then preview with:" +else + say "Next: verify with" +fi +say " cd $TARGET && DRY_RUN=true python3 sync_mirrors.py" diff --git a/mirror.env.example b/mirror.env.example new file mode 100644 index 0000000..c16e63c --- /dev/null +++ b/mirror.env.example @@ -0,0 +1,26 @@ +# Copy to mirror.env and fill in. Contains secrets -- keep at chmod 600. +# mirror.env is git-ignored on purpose; never commit the real file. + +# --- GitHub (source) --- +GITHUB_USER=your-github-username +# Fine-grained token: "All repositories" + Contents: Read-only + No expiration. +# Leave empty to mirror public repositories only. +GITHUB_TOKEN= + +# --- Gitea (target) --- +# As seen from the machine running this script. +GITEA_URL=http://127.0.0.1:3000 +# Create with: +# docker exec -u git gitea admin user generate-access-token \ +# --username --token-name github-mirror-sync \ +# --scopes write:repository,read:user +GITEA_TOKEN= +GITEA_OWNER=your-gitea-username + +# --- Behaviour --- +# How often Gitea pulls from GitHub (Go duration: 24h, 8h, 30m, ...). +MIRROR_INTERVAL=24h +INCLUDE_FORKS=false +INCLUDE_ARCHIVED=true +# Set to true to preview without changing anything. +DRY_RUN=false diff --git a/run_sync.sh b/run_sync.sh new file mode 100644 index 0000000..b2f4587 --- /dev/null +++ b/run_sync.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Cron wrapper: runs the mirror sync and keeps its log from growing unbounded +# on the Pi's SD card. All output goes to the log, so cron stays silent. +set -u + +DIR="$(dirname "$(readlink -f "$0")")" +LOG="$DIR/sync.log" + +/usr/bin/python3 "$DIR/sync_mirrors.py" >>"$LOG" 2>&1 +status=$? + +if [ -f "$LOG" ] && [ "$(wc -l <"$LOG")" -gt 2000 ]; then + tail -n 1000 "$LOG" >"$LOG.tmp" && mv "$LOG.tmp" "$LOG" +fi + +exit $status diff --git a/sync_mirrors.py b/sync_mirrors.py new file mode 100644 index 0000000..d8ff212 --- /dev/null +++ b/sync_mirrors.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +"""Mirror all GitHub repositories of a user into Gitea as pull mirrors. + +Existing mirrors are left to Gitea's own cron; this script only creates the +ones that are missing and keeps the mirror interval aligned with the config. +It never deletes anything on the Gitea side -- a mirror that vanished upstream +is exactly what a backup is supposed to keep. +""" + +import json +import os +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone + +KNOWN_KEYS = ( + "GITHUB_USER", + "GITHUB_TOKEN", + "GITEA_URL", + "GITEA_TOKEN", + "GITEA_OWNER", + "MIRROR_INTERVAL", + "INCLUDE_FORKS", + "INCLUDE_ARCHIVED", + "DRY_RUN", +) + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CONFIG_PATH = os.environ.get("MIRROR_ENV", os.path.join(SCRIPT_DIR, "mirror.env")) + +errors = 0 +last_headers = {} + + +def log(level, msg): + print("%s [%s] %s" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), level, msg), flush=True) + + +def fail(msg): + global errors + errors += 1 + log("ERROR", msg) + + +def load_config(): + cfg = { + "GITHUB_USER": "", + "GITHUB_TOKEN": "", + "GITEA_URL": "http://127.0.0.1:3000", + "GITEA_TOKEN": "", + "GITEA_OWNER": "", + "MIRROR_INTERVAL": "24h", + "INCLUDE_FORKS": "false", + "INCLUDE_ARCHIVED": "true", + "DRY_RUN": "false", + } + if os.path.exists(CONFIG_PATH): + with open(CONFIG_PATH, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + if key in KNOWN_KEYS: + cfg[key] = value.strip().strip('"').strip("'") + else: + log("WARN", "No config file at %s, relying on environment" % CONFIG_PATH) + + for key in KNOWN_KEYS: + if os.environ.get(key): + cfg[key] = os.environ[key] + + cfg["GITEA_URL"] = cfg["GITEA_URL"].rstrip("/") + return cfg + + +def truthy(value): + return str(value).strip().lower() in ("1", "true", "yes", "on") + + +def api(url, token=None, scheme="token", method="GET", payload=None): + """Return (status, parsed_body). Raises urllib.error.HTTPError on >=400.""" + data = None + headers = { + "Accept": "application/json", + "User-Agent": "gitea-github-mirror/1.0", + } + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["Authorization"] = "%s %s" % (scheme, token) + + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=180) as resp: + body = resp.read().decode("utf-8") + last_headers.clear() + last_headers.update({k.lower(): v for k, v in resp.headers.items()}) + return resp.status, (json.loads(body) if body.strip() else None) + + +def github_repos(cfg): + """All repos owned by the user; private ones included when a token is set.""" + token = cfg["GITHUB_TOKEN"] + repos = [] + page = 1 + while True: + if token: + url = ("https://api.github.com/user/repos" + "?per_page=100&affiliation=owner&page=%d" % page) + else: + url = ("https://api.github.com/users/%s/repos" + "?per_page=100&type=owner&page=%d" % (cfg["GITHUB_USER"], page)) + _, batch = api(url, token=token, scheme="Bearer") + if not batch: + break + repos.extend(batch) + if len(batch) < 100: + break + page += 1 + + if token: + # affiliation=owner still returns repos owned by orgs the user belongs to + repos = [r for r in repos + if r["owner"]["login"].lower() == cfg["GITHUB_USER"].lower()] + return repos + + +def warn_token_expiry(): + """Fine-grained PATs always expire; a silent expiry would stall the mirror. + + GitHub reports the date in a response header, so every run can check it. + """ + raw = last_headers.get("github-authentication-token-expiration") + if not raw: + return # classic token without expiry, or no token at all + text = raw.strip().replace(" UTC", "").replace("Z", "").replace("T", " ") + parsed = None + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d"): + try: + parsed = datetime.strptime(text.split(".")[0], fmt) + break + except ValueError: + continue + if parsed is None: + return + + now_utc = datetime.now(timezone.utc).replace(tzinfo=None) + days = (parsed.replace(tzinfo=None) - now_utc).days + if days < 0: + fail("GITHUB_TOKEN expired on %s - renew it" % parsed.date()) + elif days <= 14: + log("WARN", "GITHUB_TOKEN expires in %d day(s), on %s - renew it" + % (days, parsed.date())) + else: + log("INFO", "GITHUB_TOKEN valid until %s (%d days)" % (parsed.date(), days)) + + +def duration_seconds(value): + """Parse a Go-style duration ('24h', '8h0m0s') into seconds. -1 if unparseable.""" + if not value: + return -1 + total = 0 + matched = False + for amount, unit in re.findall(r"(\d+)\s*([hms])", str(value)): + matched = True + total += int(amount) * {"h": 3600, "m": 60, "s": 1}[unit] + return total if matched else -1 + + +def gitea_repo(cfg, name): + try: + _, repo = api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name), + token=cfg["GITEA_TOKEN"]) + return repo + except urllib.error.HTTPError as exc: + if exc.code == 404: + return None + raise + + +def create_mirror(cfg, repo, dry_run): + name = repo["name"] + payload = { + "clone_addr": repo["clone_url"], + "repo_name": name, + "repo_owner": cfg["GITEA_OWNER"], + "mirror": True, + "mirror_interval": cfg["MIRROR_INTERVAL"], + "private": bool(repo.get("private")), + "description": (repo.get("description") or "")[:255], + "service": "github", + # Pull mirrors only ever sync git data, so importing the rest would be a + # one-off snapshot that silently goes stale. Keep it lean on the Pi. + "wiki": False, + "issues": False, + "labels": False, + "milestones": False, + "releases": False, + "pull_requests": False, + } + if cfg["GITHUB_TOKEN"]: + payload["auth_token"] = cfg["GITHUB_TOKEN"] + + if dry_run: + log("DRY", "would mirror %s (private=%s)" % (name, payload["private"])) + return + + api("%s/api/v1/repos/migrate" % cfg["GITEA_URL"], token=cfg["GITEA_TOKEN"], + method="POST", payload=payload) + log("OK", "mirrored %s (private=%s)" % (name, payload["private"])) + + +def align_interval(cfg, name, existing, dry_run): + wanted = duration_seconds(cfg["MIRROR_INTERVAL"]) + current = duration_seconds(existing.get("mirror_interval")) + if wanted < 0 or current < 0 or wanted == current: + return + if dry_run: + log("DRY", "would set %s interval %s -> %s" + % (name, existing.get("mirror_interval"), cfg["MIRROR_INTERVAL"])) + return + api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name), + token=cfg["GITEA_TOKEN"], method="PATCH", + payload={"mirror_interval": cfg["MIRROR_INTERVAL"]}) + log("OK", "interval of %s set to %s" % (name, cfg["MIRROR_INTERVAL"])) + + +def main(): + cfg = load_config() + dry_run = truthy(cfg["DRY_RUN"]) + + missing = [k for k in ("GITHUB_USER", "GITEA_URL", "GITEA_TOKEN", "GITEA_OWNER") + if not cfg[k]] + if missing: + log("ERROR", "Missing config value(s): %s" % ", ".join(missing)) + return 2 + + if not cfg["GITHUB_TOKEN"]: + log("WARN", "GITHUB_TOKEN is empty - private repositories will be skipped") + + try: + repos = github_repos(cfg) + except urllib.error.HTTPError as exc: + log("ERROR", "GitHub API %s: %s" % (exc.code, exc.read().decode("utf-8", "replace")[:300])) + return 2 + except Exception as exc: + log("ERROR", "GitHub API unreachable: %s" % exc) + return 2 + + warn_token_expiry() # must run before any Gitea call overwrites the headers + + if not truthy(cfg["INCLUDE_FORKS"]): + repos = [r for r in repos if not r.get("fork")] + if not truthy(cfg["INCLUDE_ARCHIVED"]): + repos = [r for r in repos if not r.get("archived")] + + private_count = sum(1 for r in repos if r.get("private")) + log("INFO", "GitHub: %d repositor%s to consider (%d public, %d private)%s" + % (len(repos), "y" if len(repos) == 1 else "ies", + len(repos) - private_count, private_count, + " (DRY RUN)" if dry_run else "")) + + # A fine-grained token defaults to "Public repositories", which looks exactly + # like an account without private repos. Name the likely cause explicitly. + if cfg["GITHUB_TOKEN"] and private_count == 0: + log("WARN", "Token is set but GitHub reports no private repositories - " + "check that its repository access is 'All repositories', " + "not 'Public repositories'") + + created = skipped = 0 + for repo in sorted(repos, key=lambda r: r["name"].lower()): + name = repo["name"] + try: + existing = gitea_repo(cfg, name) + except Exception as exc: + fail("%s: cannot query Gitea: %s" % (name, exc)) + continue + + if existing is None: + try: + create_mirror(cfg, repo, dry_run) + created += 1 + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", "replace")[:300] + fail("%s: migrate failed (HTTP %s): %s" % (name, exc.code, detail)) + except Exception as exc: + fail("%s: migrate failed: %s" % (name, exc)) + continue + + skipped += 1 + if not existing.get("mirror"): + log("WARN", "%s exists in Gitea but is not a mirror - left untouched" % name) + continue + try: + align_interval(cfg, name, existing, dry_run) + except Exception as exc: + fail("%s: could not update mirror interval: %s" % (name, exc)) + + github_names = {r["name"].lower() for r in repos} + try: + page, orphans = 1, [] + while True: + _, batch = api("%s/api/v1/user/repos?limit=50&page=%d" % (cfg["GITEA_URL"], page), + token=cfg["GITEA_TOKEN"]) + if not batch: + break + orphans.extend( + r["name"] for r in batch + if r.get("mirror") + and r["owner"]["login"].lower() == cfg["GITEA_OWNER"].lower() + and r["name"].lower() not in github_names + ) + if len(batch) < 50: + break + page += 1 + if orphans: + log("WARN", "Mirrors with no GitHub counterpart (renamed or deleted upstream): %s" + % ", ".join(sorted(orphans))) + except Exception as exc: + log("WARN", "Could not check for orphaned mirrors: %s" % exc) + + log("INFO", "Done: %d created, %d already present, %d error(s)" % (created, skipped, errors)) + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(main())