mirror of
https://github.com/Andre-Bzk/github-to-gitea-mirror.git
synced 2026-11-15 09:33:13 +01:00
Andre
6ff4da99e4
The old name left the mirror direction open — it could just as well be read as Gitea being the source. Naming source and target in that order removes the ambiguity for anyone setting this up, and matches the title the README already carried. The name is not just internal: it is the install directory and the identifier both APIs record in their request logs, so an inconsistent name would stay visible in places that are awkward to correct after the fact. Existing installations are not migrated. The old directory and its cron entry survive an installer re-run and would sync a second time in parallel, so they need to be removed by hand.
702 lines
26 KiB
Python
702 lines
26 KiB
Python
#!/usr/bin/env python3
|
|
"""Mirror all GitHub repositories of a user into Gitea as pull mirrors.
|
|
|
|
Existing git mirrors are left to Gitea's own cron. This script creates the ones
|
|
that are missing, aligns their interval, and synchronizes GitHub release
|
|
metadata and assets because Gitea's native mirror does not. It never deletes
|
|
anything on the Gitea side -- retained history is the point of a backup.
|
|
"""
|
|
|
|
import json
|
|
import http.client
|
|
import io
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
KNOWN_KEYS = (
|
|
"GITHUB_USER",
|
|
"GITHUB_TOKEN",
|
|
"GITEA_URL",
|
|
"GITEA_TOKEN",
|
|
"GITEA_OWNER",
|
|
"MIRROR_INTERVAL",
|
|
"INCLUDE_FORKS",
|
|
"INCLUDE_ARCHIVED",
|
|
"SYNC_RELEASES",
|
|
"SYNC_RELEASE_ASSETS",
|
|
"DRY_RUN",
|
|
)
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
CONFIG_PATH = os.environ.get("MIRROR_ENV", os.path.join(SCRIPT_DIR, "mirror.env"))
|
|
|
|
errors = 0
|
|
last_headers = {}
|
|
|
|
|
|
def log(level, msg):
|
|
print("%s [%s] %s" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), level, msg), flush=True)
|
|
|
|
|
|
def fail(msg):
|
|
global errors
|
|
errors += 1
|
|
log("ERROR", msg)
|
|
|
|
|
|
def load_config():
|
|
cfg = {
|
|
"GITHUB_USER": "",
|
|
"GITHUB_TOKEN": "",
|
|
"GITEA_URL": "http://127.0.0.1:3000",
|
|
"GITEA_TOKEN": "",
|
|
"GITEA_OWNER": "",
|
|
"MIRROR_INTERVAL": "24h",
|
|
"INCLUDE_FORKS": "false",
|
|
"INCLUDE_ARCHIVED": "true",
|
|
"SYNC_RELEASES": "true",
|
|
"SYNC_RELEASE_ASSETS": "true",
|
|
"DRY_RUN": "false",
|
|
}
|
|
if os.path.exists(CONFIG_PATH):
|
|
with open(CONFIG_PATH, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
key = key.strip()
|
|
if key in KNOWN_KEYS:
|
|
cfg[key] = value.strip().strip('"').strip("'")
|
|
else:
|
|
log("WARN", "No config file at %s, relying on environment" % CONFIG_PATH)
|
|
|
|
for key in KNOWN_KEYS:
|
|
if os.environ.get(key):
|
|
cfg[key] = os.environ[key]
|
|
|
|
cfg["GITEA_URL"] = cfg["GITEA_URL"].rstrip("/")
|
|
return cfg
|
|
|
|
|
|
def truthy(value):
|
|
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
|
|
|
|
|
def api(url, token=None, scheme="token", method="GET", payload=None):
|
|
"""Return (status, parsed_body). Raises urllib.error.HTTPError on >=400."""
|
|
data = None
|
|
headers = {
|
|
"Accept": "application/json",
|
|
"User-Agent": "github-to-gitea-mirror/1.0",
|
|
}
|
|
if payload is not None:
|
|
data = json.dumps(payload).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = "%s %s" % (scheme, token)
|
|
|
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
with urllib.request.urlopen(req, timeout=180) as resp:
|
|
body = resp.read().decode("utf-8")
|
|
last_headers.clear()
|
|
last_headers.update({k.lower(): v for k, v in resp.headers.items()})
|
|
return resp.status, (json.loads(body) if body.strip() else None)
|
|
|
|
|
|
def path_part(value):
|
|
"""Quote one URL path component, including slashes used inside tag names."""
|
|
return urllib.parse.quote(str(value), safe="")
|
|
|
|
|
|
class SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
"""Do not forward API credentials to a cross-host download redirect."""
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
if redirected is None:
|
|
return None
|
|
old_host = urllib.parse.urlsplit(req.full_url).netloc.lower()
|
|
new_host = urllib.parse.urlsplit(newurl).netloc.lower()
|
|
if old_host != new_host:
|
|
redirected.remove_header("Authorization")
|
|
return redirected
|
|
|
|
|
|
safe_download_opener = urllib.request.build_opener(SafeRedirectHandler())
|
|
|
|
|
|
def download_github_asset(cfg, asset):
|
|
"""Download a release asset into a temporary file without buffering it in RAM."""
|
|
headers = {
|
|
"Accept": "application/octet-stream",
|
|
"User-Agent": "github-to-gitea-mirror/1.0",
|
|
}
|
|
if cfg["GITHUB_TOKEN"]:
|
|
headers["Authorization"] = "Bearer %s" % cfg["GITHUB_TOKEN"]
|
|
req = urllib.request.Request(asset["url"], headers=headers)
|
|
tmp = tempfile.TemporaryFile()
|
|
size = 0
|
|
try:
|
|
with safe_download_opener.open(req, timeout=180) as resp:
|
|
while True:
|
|
chunk = resp.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
tmp.write(chunk)
|
|
size += len(chunk)
|
|
expected = int(asset.get("size") or 0)
|
|
if expected and size != expected:
|
|
raise IOError("downloaded %d bytes, expected %d" % (size, expected))
|
|
tmp.seek(0)
|
|
return tmp, size
|
|
except Exception:
|
|
tmp.close()
|
|
raise
|
|
|
|
|
|
def upload_release_asset(cfg, repo_name, release_id, asset, source, size):
|
|
"""Stream a temporary file to Gitea as multipart/form-data."""
|
|
query = urllib.parse.urlencode({"name": asset["name"]})
|
|
url = ("%s/api/v1/repos/%s/%s/releases/%s/assets?%s"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]), path_part(repo_name),
|
|
release_id, query))
|
|
parsed = urllib.parse.urlsplit(url)
|
|
boundary = "----github-to-gitea-mirror-%s" % uuid.uuid4().hex
|
|
safe_name = str(asset["name"]).replace("\\", "_").replace('"', "_")
|
|
safe_name = safe_name.replace("\r", "_").replace("\n", "_")
|
|
content_type = asset.get("content_type") or "application/octet-stream"
|
|
content_type = str(content_type).replace("\r", "").replace("\n", "")
|
|
preamble = (
|
|
"--%s\r\n"
|
|
"Content-Disposition: form-data; name=\"attachment\"; filename=\"%s\"\r\n"
|
|
"Content-Type: %s\r\n\r\n" % (boundary, safe_name, content_type)
|
|
).encode("utf-8")
|
|
ending = ("\r\n--%s--\r\n" % boundary).encode("ascii")
|
|
request_path = parsed.path + (("?" + parsed.query) if parsed.query else "")
|
|
|
|
if parsed.scheme == "https":
|
|
conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=180)
|
|
elif parsed.scheme == "http":
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=180)
|
|
else:
|
|
raise ValueError("Unsupported Gitea URL scheme: %s" % parsed.scheme)
|
|
|
|
try:
|
|
conn.putrequest("POST", request_path)
|
|
conn.putheader("Authorization", "token %s" % cfg["GITEA_TOKEN"])
|
|
conn.putheader("Accept", "application/json")
|
|
conn.putheader("Content-Type", "multipart/form-data; boundary=%s" % boundary)
|
|
conn.putheader("Content-Length", str(len(preamble) + size + len(ending)))
|
|
conn.putheader("User-Agent", "github-to-gitea-mirror/1.0")
|
|
conn.endheaders()
|
|
conn.send(preamble)
|
|
while True:
|
|
chunk = source.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
conn.send(chunk)
|
|
conn.send(ending)
|
|
response = conn.getresponse()
|
|
body = response.read()
|
|
if response.status >= 400:
|
|
raise urllib.error.HTTPError(
|
|
url, response.status, response.reason, response.headers, io.BytesIO(body))
|
|
return json.loads(body.decode("utf-8")) if body.strip() else None
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def github_repos(cfg):
|
|
"""All repos owned by the user; private ones included when a token is set."""
|
|
token = cfg["GITHUB_TOKEN"]
|
|
repos = []
|
|
page = 1
|
|
while True:
|
|
if token:
|
|
url = ("https://api.github.com/user/repos"
|
|
"?per_page=100&affiliation=owner&page=%d" % page)
|
|
else:
|
|
url = ("https://api.github.com/users/%s/repos"
|
|
"?per_page=100&type=owner&page=%d" % (cfg["GITHUB_USER"], page))
|
|
_, batch = api(url, token=token, scheme="Bearer")
|
|
if not batch:
|
|
break
|
|
repos.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
|
|
if token:
|
|
# affiliation=owner still returns repos owned by orgs the user belongs to
|
|
repos = [r for r in repos
|
|
if r["owner"]["login"].lower() == cfg["GITHUB_USER"].lower()]
|
|
return repos
|
|
|
|
|
|
def warn_token_expiry():
|
|
"""Fine-grained PATs always expire; a silent expiry would stall the mirror.
|
|
|
|
GitHub reports the date in a response header, so every run can check it.
|
|
"""
|
|
raw = last_headers.get("github-authentication-token-expiration")
|
|
if not raw:
|
|
return # classic token without expiry, or no token at all
|
|
text = raw.strip().replace(" UTC", "").replace("Z", "").replace("T", " ")
|
|
parsed = None
|
|
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d"):
|
|
try:
|
|
parsed = datetime.strptime(text.split(".")[0], fmt)
|
|
break
|
|
except ValueError:
|
|
continue
|
|
if parsed is None:
|
|
return
|
|
|
|
now_utc = datetime.now(timezone.utc).replace(tzinfo=None)
|
|
days = (parsed.replace(tzinfo=None) - now_utc).days
|
|
if days < 0:
|
|
fail("GITHUB_TOKEN expired on %s - renew it" % parsed.date())
|
|
elif days <= 14:
|
|
log("WARN", "GITHUB_TOKEN expires in %d day(s), on %s - renew it"
|
|
% (days, parsed.date()))
|
|
else:
|
|
log("INFO", "GITHUB_TOKEN valid until %s (%d days)" % (parsed.date(), days))
|
|
|
|
|
|
def duration_seconds(value):
|
|
"""Parse a Go-style duration ('24h', '8h0m0s') into seconds. -1 if unparseable."""
|
|
if not value:
|
|
return -1
|
|
total = 0
|
|
matched = False
|
|
for amount, unit in re.findall(r"(\d+)\s*([hms])", str(value)):
|
|
matched = True
|
|
total += int(amount) * {"h": 3600, "m": 60, "s": 1}[unit]
|
|
return total if matched else -1
|
|
|
|
|
|
def gitea_repo(cfg, name):
|
|
try:
|
|
_, repo = api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
|
|
token=cfg["GITEA_TOKEN"])
|
|
return repo
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
raise
|
|
|
|
|
|
def create_mirror(cfg, repo, dry_run):
|
|
name = repo["name"]
|
|
payload = {
|
|
"clone_addr": repo["clone_url"],
|
|
"repo_name": name,
|
|
"repo_owner": cfg["GITEA_OWNER"],
|
|
"mirror": True,
|
|
"mirror_interval": cfg["MIRROR_INTERVAL"],
|
|
"private": bool(repo.get("private")),
|
|
"description": (repo.get("description") or "")[:255],
|
|
"service": "github",
|
|
# Pull mirrors only ever sync git data, so importing the rest would be a
|
|
# one-off snapshot that silently goes stale. Keep it lean on the Pi.
|
|
"wiki": False,
|
|
"issues": False,
|
|
"labels": False,
|
|
"milestones": False,
|
|
"releases": False,
|
|
"pull_requests": False,
|
|
}
|
|
if cfg["GITHUB_TOKEN"]:
|
|
payload["auth_token"] = cfg["GITHUB_TOKEN"]
|
|
|
|
if dry_run:
|
|
log("DRY", "would mirror %s (private=%s)" % (name, payload["private"]))
|
|
return
|
|
|
|
api("%s/api/v1/repos/migrate" % cfg["GITEA_URL"], token=cfg["GITEA_TOKEN"],
|
|
method="POST", payload=payload)
|
|
log("OK", "mirrored %s (private=%s)" % (name, payload["private"]))
|
|
|
|
|
|
def align_interval(cfg, name, existing, dry_run):
|
|
wanted = duration_seconds(cfg["MIRROR_INTERVAL"])
|
|
current = duration_seconds(existing.get("mirror_interval"))
|
|
if wanted < 0 or current < 0 or wanted == current:
|
|
return
|
|
if dry_run:
|
|
log("DRY", "would set %s interval %s -> %s"
|
|
% (name, existing.get("mirror_interval"), cfg["MIRROR_INTERVAL"]))
|
|
return
|
|
api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
|
|
token=cfg["GITEA_TOKEN"], method="PATCH",
|
|
payload={"mirror_interval": cfg["MIRROR_INTERVAL"]})
|
|
log("OK", "interval of %s set to %s" % (name, cfg["MIRROR_INTERVAL"]))
|
|
|
|
|
|
def github_releases(cfg, repo_name):
|
|
releases = []
|
|
page = 1
|
|
while True:
|
|
url = ("https://api.github.com/repos/%s/%s/releases?per_page=100&page=%d"
|
|
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), page))
|
|
_, batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer")
|
|
if not batch:
|
|
break
|
|
releases.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
return releases
|
|
|
|
|
|
def github_release_assets(cfg, repo_name, release_id):
|
|
assets = []
|
|
page = 1
|
|
while True:
|
|
url = ("https://api.github.com/repos/%s/%s/releases/%s/assets"
|
|
"?per_page=100&page=%d"
|
|
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), release_id, page))
|
|
_, batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer")
|
|
if not batch:
|
|
break
|
|
assets.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
return assets
|
|
|
|
|
|
def gitea_releases(cfg, repo_name):
|
|
releases = []
|
|
page = 1
|
|
while True:
|
|
url = ("%s/api/v1/repos/%s/%s/releases?limit=50&page=%d"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), page))
|
|
_, batch = api(url, token=cfg["GITEA_TOKEN"])
|
|
if not batch:
|
|
break
|
|
releases.extend(batch)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
return releases
|
|
|
|
|
|
def gitea_tags(cfg, repo_name):
|
|
tags = set()
|
|
page = 1
|
|
while True:
|
|
url = ("%s/api/v1/repos/%s/%s/tags?limit=50&page=%d"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), page))
|
|
_, batch = api(url, token=cfg["GITEA_TOKEN"])
|
|
if not batch:
|
|
break
|
|
tags.update(tag["name"] for tag in batch)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
return tags
|
|
|
|
|
|
def release_payload(release):
|
|
return {
|
|
"tag_name": release["tag_name"],
|
|
"target_commitish": release.get("target_commitish") or "",
|
|
"name": release.get("name") or "",
|
|
"body": release.get("body") or "",
|
|
"draft": bool(release.get("draft")),
|
|
"prerelease": bool(release.get("prerelease")),
|
|
}
|
|
|
|
|
|
def release_needs_update(source, target):
|
|
"""Return true for fields Gitea's PATCH endpoint can actually change."""
|
|
wanted = release_payload(source)
|
|
if bool(target.get("draft")) != wanted["draft"]:
|
|
return True
|
|
if bool(target.get("prerelease")) != wanted["prerelease"]:
|
|
return True
|
|
# Gitea ignores empty strings in release PATCH requests. Retaining old text
|
|
# also follows this script's policy of never deleting backup information.
|
|
if wanted["name"] and (target.get("name") or "") != wanted["name"]:
|
|
return True
|
|
if wanted["body"] and (target.get("body") or "") != wanted["body"]:
|
|
return True
|
|
return False
|
|
|
|
|
|
def release_update_payload(source):
|
|
wanted = release_payload(source)
|
|
payload = {
|
|
"draft": wanted["draft"],
|
|
"prerelease": wanted["prerelease"],
|
|
}
|
|
for key in ("target_commitish", "name", "body"):
|
|
if wanted[key]:
|
|
payload[key] = wanted[key]
|
|
return payload
|
|
|
|
|
|
def sync_release_assets(cfg, repo_name, source_release, target_release, dry_run):
|
|
copied = preserved = 0
|
|
source_assets = github_release_assets(cfg, repo_name, source_release["id"])
|
|
target_assets = {asset["name"]: asset for asset in target_release.get("assets", [])}
|
|
|
|
for asset in source_assets:
|
|
name = asset["name"]
|
|
existing = target_assets.get(name)
|
|
if existing is not None:
|
|
source_size = int(asset.get("size") or 0)
|
|
target_size = int(existing.get("size") or 0)
|
|
if source_size and target_size and source_size != target_size:
|
|
preserved += 1
|
|
log("WARN", "%s release %s asset %s differs (%d vs %d bytes) - "
|
|
"Gitea copy left untouched"
|
|
% (repo_name, source_release["tag_name"], name,
|
|
source_size, target_size))
|
|
continue
|
|
|
|
if dry_run:
|
|
log("DRY", "would copy %s release %s asset %s (%d bytes)"
|
|
% (repo_name, source_release["tag_name"], name,
|
|
int(asset.get("size") or 0)))
|
|
copied += 1
|
|
continue
|
|
|
|
tmp = None
|
|
try:
|
|
tmp, size = download_github_asset(cfg, asset)
|
|
upload_release_asset(cfg, repo_name, target_release["id"], asset, tmp, size)
|
|
copied += 1
|
|
log("OK", "copied %s release %s asset %s (%d bytes)"
|
|
% (repo_name, source_release["tag_name"], name, size))
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
fail("%s release %s asset %s failed (HTTP %s): %s"
|
|
% (repo_name, source_release["tag_name"], name, exc.code, detail))
|
|
except Exception as exc:
|
|
fail("%s release %s asset %s failed: %s"
|
|
% (repo_name, source_release["tag_name"], name, exc))
|
|
finally:
|
|
if tmp is not None:
|
|
tmp.close()
|
|
return copied, preserved
|
|
|
|
|
|
def sync_releases(cfg, repo_name, dry_run):
|
|
stats = {"created": 0, "updated": 0, "assets": 0, "preserved": 0, "waiting": 0}
|
|
source_releases = github_releases(cfg, repo_name)
|
|
target_releases = gitea_releases(cfg, repo_name)
|
|
by_tag = {release["tag_name"]: release for release in target_releases}
|
|
tags_to_check = [
|
|
release["tag_name"] for release in source_releases
|
|
if not release.get("draft")
|
|
and (release["tag_name"] not in by_tag
|
|
or by_tag[release["tag_name"]].get("draft"))
|
|
]
|
|
available_tags = gitea_tags(cfg, repo_name) if tags_to_check else set()
|
|
|
|
source_tags = {release["tag_name"] for release in source_releases}
|
|
orphaned = sorted(tag for tag in by_tag if tag not in source_tags)
|
|
if orphaned:
|
|
log("WARN", "%s Gitea release(s) with no GitHub counterpart left untouched: %s"
|
|
% (repo_name, ", ".join(orphaned)))
|
|
|
|
for source in source_releases:
|
|
tag = source["tag_name"]
|
|
target = by_tag.get(tag)
|
|
needs_mirrored_tag = (not source.get("draft")
|
|
and (target is None or target.get("draft")))
|
|
if needs_mirrored_tag and tag not in available_tags:
|
|
stats["waiting"] += 1
|
|
log("WARN", "%s release %s waits for its tag to reach the Gitea mirror"
|
|
% (repo_name, tag))
|
|
continue
|
|
|
|
try:
|
|
if target is None:
|
|
if dry_run:
|
|
target = {"id": source["id"], "tag_name": tag, "assets": []}
|
|
log("DRY", "would create %s release %s" % (repo_name, tag))
|
|
else:
|
|
_, target = api(
|
|
"%s/api/v1/repos/%s/%s/releases"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name)),
|
|
token=cfg["GITEA_TOKEN"], method="POST",
|
|
payload=release_payload(source))
|
|
log("OK", "created %s release %s" % (repo_name, tag))
|
|
stats["created"] += 1
|
|
elif release_needs_update(source, target):
|
|
if dry_run:
|
|
log("DRY", "would update %s release %s" % (repo_name, tag))
|
|
else:
|
|
_, target = api(
|
|
"%s/api/v1/repos/%s/%s/releases/%s"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), target["id"]),
|
|
token=cfg["GITEA_TOKEN"], method="PATCH",
|
|
payload=release_update_payload(source))
|
|
log("OK", "updated %s release %s" % (repo_name, tag))
|
|
stats["updated"] += 1
|
|
|
|
if truthy(cfg["SYNC_RELEASE_ASSETS"]):
|
|
copied, preserved = sync_release_assets(
|
|
cfg, repo_name, source, target, dry_run)
|
|
stats["assets"] += copied
|
|
stats["preserved"] += preserved
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
fail("%s release %s failed (HTTP %s): %s"
|
|
% (repo_name, tag, exc.code, detail))
|
|
except Exception as exc:
|
|
fail("%s release %s failed: %s" % (repo_name, tag, exc))
|
|
return stats
|
|
|
|
|
|
def main():
|
|
cfg = load_config()
|
|
dry_run = truthy(cfg["DRY_RUN"])
|
|
|
|
missing = [k for k in ("GITHUB_USER", "GITEA_URL", "GITEA_TOKEN", "GITEA_OWNER")
|
|
if not cfg[k]]
|
|
if missing:
|
|
log("ERROR", "Missing config value(s): %s" % ", ".join(missing))
|
|
return 2
|
|
|
|
if not cfg["GITHUB_TOKEN"]:
|
|
log("WARN", "GITHUB_TOKEN is empty - private repositories will be skipped")
|
|
|
|
try:
|
|
repos = github_repos(cfg)
|
|
except urllib.error.HTTPError as exc:
|
|
log("ERROR", "GitHub API %s: %s" % (exc.code, exc.read().decode("utf-8", "replace")[:300]))
|
|
return 2
|
|
except Exception as exc:
|
|
log("ERROR", "GitHub API unreachable: %s" % exc)
|
|
return 2
|
|
|
|
warn_token_expiry() # must run before any Gitea call overwrites the headers
|
|
|
|
if not truthy(cfg["INCLUDE_FORKS"]):
|
|
repos = [r for r in repos if not r.get("fork")]
|
|
if not truthy(cfg["INCLUDE_ARCHIVED"]):
|
|
repos = [r for r in repos if not r.get("archived")]
|
|
|
|
private_count = sum(1 for r in repos if r.get("private"))
|
|
log("INFO", "GitHub: %d repositor%s to consider (%d public, %d private)%s"
|
|
% (len(repos), "y" if len(repos) == 1 else "ies",
|
|
len(repos) - private_count, private_count,
|
|
" (DRY RUN)" if dry_run else ""))
|
|
|
|
# A fine-grained token defaults to "Public repositories", which looks exactly
|
|
# like an account without private repos. Name the likely cause explicitly.
|
|
if cfg["GITHUB_TOKEN"] and private_count == 0:
|
|
log("WARN", "Token is set but GitHub reports no private repositories - "
|
|
"check that its repository access is 'All repositories', "
|
|
"not 'Public repositories'")
|
|
|
|
created = skipped = 0
|
|
release_totals = {"created": 0, "updated": 0, "assets": 0,
|
|
"preserved": 0, "waiting": 0}
|
|
for repo in sorted(repos, key=lambda r: r["name"].lower()):
|
|
name = repo["name"]
|
|
try:
|
|
existing = gitea_repo(cfg, name)
|
|
except Exception as exc:
|
|
fail("%s: cannot query Gitea: %s" % (name, exc))
|
|
continue
|
|
|
|
if existing is None:
|
|
mirror_created = False
|
|
try:
|
|
create_mirror(cfg, repo, dry_run)
|
|
created += 1
|
|
mirror_created = True
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
fail("%s: migrate failed (HTTP %s): %s" % (name, exc.code, detail))
|
|
except Exception as exc:
|
|
fail("%s: migrate failed: %s" % (name, exc))
|
|
|
|
if mirror_created and truthy(cfg["SYNC_RELEASES"]):
|
|
try:
|
|
if dry_run:
|
|
count = len(github_releases(cfg, name))
|
|
if count:
|
|
log("DRY", "would synchronize %d release(s) after creating %s"
|
|
% (count, name))
|
|
else:
|
|
stats = sync_releases(cfg, name, dry_run)
|
|
for key in release_totals:
|
|
release_totals[key] += stats[key]
|
|
except Exception as exc:
|
|
fail("%s: could not synchronize releases: %s" % (name, exc))
|
|
continue
|
|
|
|
skipped += 1
|
|
if not existing.get("mirror"):
|
|
log("WARN", "%s exists in Gitea but is not a mirror - left untouched" % name)
|
|
continue
|
|
try:
|
|
align_interval(cfg, name, existing, dry_run)
|
|
except Exception as exc:
|
|
fail("%s: could not update mirror interval: %s" % (name, exc))
|
|
|
|
if truthy(cfg["SYNC_RELEASES"]):
|
|
try:
|
|
stats = sync_releases(cfg, name, dry_run)
|
|
for key in release_totals:
|
|
release_totals[key] += stats[key]
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
fail("%s: release sync failed (HTTP %s): %s"
|
|
% (name, exc.code, detail))
|
|
except Exception as exc:
|
|
fail("%s: release sync failed: %s" % (name, exc))
|
|
|
|
github_names = {r["name"].lower() for r in repos}
|
|
try:
|
|
page, orphans = 1, []
|
|
while True:
|
|
_, batch = api("%s/api/v1/user/repos?limit=50&page=%d" % (cfg["GITEA_URL"], page),
|
|
token=cfg["GITEA_TOKEN"])
|
|
if not batch:
|
|
break
|
|
orphans.extend(
|
|
r["name"] for r in batch
|
|
if r.get("mirror")
|
|
and r["owner"]["login"].lower() == cfg["GITEA_OWNER"].lower()
|
|
and r["name"].lower() not in github_names
|
|
)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
if orphans:
|
|
log("WARN", "Mirrors with no GitHub counterpart (renamed or deleted upstream): %s"
|
|
% ", ".join(sorted(orphans)))
|
|
except Exception as exc:
|
|
log("WARN", "Could not check for orphaned mirrors: %s" % exc)
|
|
|
|
log("INFO", "Done: %d mirror(s) created, %d already present; releases: "
|
|
"%d created, %d updated, %d asset(s) copied, %d preserved, "
|
|
"%d waiting for tags; %d error(s)"
|
|
% (created, skipped, release_totals["created"], release_totals["updated"],
|
|
release_totals["assets"], release_totals["preserved"],
|
|
release_totals["waiting"], errors))
|
|
return 1 if errors else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|