mirror of
https://github.com/Andre-Bzk/github-to-gitea-mirror.git
synced 2026-11-15 09:33:13 +01:00
Andre
074fd22eff
The token expiry check read response headers that an earlier, unrelated request had left behind in module state. Its correctness depended on call order that nothing but a comment enforced, so inserting or reordering a request would have silently checked the wrong response, or none at all. The exit code had the mirrored problem: any function in the call tree could raise a shared error count, so nothing at the top level revealed which steps could contribute to a failing run. Both now travel as return values. The ordering constraint becomes a data dependency the interpreter enforces instead of a convention the next reader has to know about.
724 lines
27 KiB
Python
724 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
"""Mirror all GitHub repositories of a user into Gitea as pull mirrors.
|
|
|
|
Existing git mirrors are left to Gitea's own cron. This script creates the ones
|
|
that are missing, aligns their interval, and synchronizes GitHub release
|
|
metadata and assets because Gitea's native mirror does not. It never deletes
|
|
anything on the Gitea side -- retained history is the point of a backup.
|
|
"""
|
|
|
|
import json
|
|
import http.client
|
|
import io
|
|
import os
|
|
import re
|
|
import sys
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from collections import namedtuple
|
|
from datetime import datetime, timezone
|
|
|
|
KNOWN_KEYS = (
|
|
"GITHUB_USER",
|
|
"GITHUB_TOKEN",
|
|
"GITEA_URL",
|
|
"GITEA_TOKEN",
|
|
"GITEA_OWNER",
|
|
"MIRROR_INTERVAL",
|
|
"INCLUDE_FORKS",
|
|
"INCLUDE_ARCHIVED",
|
|
"SYNC_RELEASES",
|
|
"SYNC_RELEASE_ASSETS",
|
|
"DRY_RUN",
|
|
)
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
CONFIG_PATH = os.environ.get("MIRROR_ENV", os.path.join(SCRIPT_DIR, "mirror.env"))
|
|
|
|
Response = namedtuple("Response", "status body headers")
|
|
|
|
|
|
def log(level, msg):
|
|
print("%s [%s] %s" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), level, msg), flush=True)
|
|
|
|
|
|
def load_config():
|
|
cfg = {
|
|
"GITHUB_USER": "",
|
|
"GITHUB_TOKEN": "",
|
|
"GITEA_URL": "http://127.0.0.1:3000",
|
|
"GITEA_TOKEN": "",
|
|
"GITEA_OWNER": "",
|
|
"MIRROR_INTERVAL": "24h",
|
|
"INCLUDE_FORKS": "false",
|
|
"INCLUDE_ARCHIVED": "true",
|
|
"SYNC_RELEASES": "true",
|
|
"SYNC_RELEASE_ASSETS": "true",
|
|
"DRY_RUN": "false",
|
|
}
|
|
if os.path.exists(CONFIG_PATH):
|
|
with open(CONFIG_PATH, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
key = key.strip()
|
|
if key in KNOWN_KEYS:
|
|
cfg[key] = value.strip().strip('"').strip("'")
|
|
else:
|
|
log("WARN", "No config file at %s, relying on environment" % CONFIG_PATH)
|
|
|
|
for key in KNOWN_KEYS:
|
|
if os.environ.get(key):
|
|
cfg[key] = os.environ[key]
|
|
|
|
cfg["GITEA_URL"] = cfg["GITEA_URL"].rstrip("/")
|
|
return cfg
|
|
|
|
|
|
def truthy(value):
|
|
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
|
|
|
|
|
def api(url, token=None, scheme="token", method="GET", payload=None):
|
|
"""Return a Response. Raises urllib.error.HTTPError on >=400."""
|
|
data = None
|
|
headers = {
|
|
"Accept": "application/json",
|
|
"User-Agent": "github-to-gitea-mirror/1.0",
|
|
}
|
|
if payload is not None:
|
|
data = json.dumps(payload).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = "%s %s" % (scheme, token)
|
|
|
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
with urllib.request.urlopen(req, timeout=180) as resp:
|
|
body = resp.read().decode("utf-8")
|
|
return Response(
|
|
resp.status,
|
|
json.loads(body) if body.strip() else None,
|
|
{k.lower(): v for k, v in resp.headers.items()},
|
|
)
|
|
|
|
|
|
def path_part(value):
|
|
"""Quote one URL path component, including slashes used inside tag names."""
|
|
return urllib.parse.quote(str(value), safe="")
|
|
|
|
|
|
class SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
"""Do not forward API credentials to a cross-host download redirect."""
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
if redirected is None:
|
|
return None
|
|
old_host = urllib.parse.urlsplit(req.full_url).netloc.lower()
|
|
new_host = urllib.parse.urlsplit(newurl).netloc.lower()
|
|
if old_host != new_host:
|
|
redirected.remove_header("Authorization")
|
|
return redirected
|
|
|
|
|
|
safe_download_opener = urllib.request.build_opener(SafeRedirectHandler())
|
|
|
|
|
|
def download_github_asset(cfg, asset):
|
|
"""Download a release asset into a temporary file without buffering it in RAM."""
|
|
headers = {
|
|
"Accept": "application/octet-stream",
|
|
"User-Agent": "github-to-gitea-mirror/1.0",
|
|
}
|
|
if cfg["GITHUB_TOKEN"]:
|
|
headers["Authorization"] = "Bearer %s" % cfg["GITHUB_TOKEN"]
|
|
req = urllib.request.Request(asset["url"], headers=headers)
|
|
tmp = tempfile.TemporaryFile()
|
|
size = 0
|
|
try:
|
|
with safe_download_opener.open(req, timeout=180) as resp:
|
|
while True:
|
|
chunk = resp.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
tmp.write(chunk)
|
|
size += len(chunk)
|
|
expected = int(asset.get("size") or 0)
|
|
if expected and size != expected:
|
|
raise IOError("downloaded %d bytes, expected %d" % (size, expected))
|
|
tmp.seek(0)
|
|
return tmp, size
|
|
except Exception:
|
|
tmp.close()
|
|
raise
|
|
|
|
|
|
def upload_release_asset(cfg, repo_name, release_id, asset, source, size):
|
|
"""Stream a temporary file to Gitea as multipart/form-data."""
|
|
query = urllib.parse.urlencode({"name": asset["name"]})
|
|
url = ("%s/api/v1/repos/%s/%s/releases/%s/assets?%s"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]), path_part(repo_name),
|
|
release_id, query))
|
|
parsed = urllib.parse.urlsplit(url)
|
|
boundary = "----github-to-gitea-mirror-%s" % uuid.uuid4().hex
|
|
safe_name = str(asset["name"]).replace("\\", "_").replace('"', "_")
|
|
safe_name = safe_name.replace("\r", "_").replace("\n", "_")
|
|
content_type = asset.get("content_type") or "application/octet-stream"
|
|
content_type = str(content_type).replace("\r", "").replace("\n", "")
|
|
preamble = (
|
|
"--%s\r\n"
|
|
"Content-Disposition: form-data; name=\"attachment\"; filename=\"%s\"\r\n"
|
|
"Content-Type: %s\r\n\r\n" % (boundary, safe_name, content_type)
|
|
).encode("utf-8")
|
|
ending = ("\r\n--%s--\r\n" % boundary).encode("ascii")
|
|
request_path = parsed.path + (("?" + parsed.query) if parsed.query else "")
|
|
|
|
if parsed.scheme == "https":
|
|
conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=180)
|
|
elif parsed.scheme == "http":
|
|
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=180)
|
|
else:
|
|
raise ValueError("Unsupported Gitea URL scheme: %s" % parsed.scheme)
|
|
|
|
try:
|
|
conn.putrequest("POST", request_path)
|
|
conn.putheader("Authorization", "token %s" % cfg["GITEA_TOKEN"])
|
|
conn.putheader("Accept", "application/json")
|
|
conn.putheader("Content-Type", "multipart/form-data; boundary=%s" % boundary)
|
|
conn.putheader("Content-Length", str(len(preamble) + size + len(ending)))
|
|
conn.putheader("User-Agent", "github-to-gitea-mirror/1.0")
|
|
conn.endheaders()
|
|
conn.send(preamble)
|
|
while True:
|
|
chunk = source.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
conn.send(chunk)
|
|
conn.send(ending)
|
|
response = conn.getresponse()
|
|
body = response.read()
|
|
if response.status >= 400:
|
|
raise urllib.error.HTTPError(
|
|
url, response.status, response.reason, response.headers, io.BytesIO(body))
|
|
return json.loads(body.decode("utf-8")) if body.strip() else None
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def github_repos(cfg):
|
|
"""Return (repos, headers of the last GitHub response).
|
|
|
|
The headers travel with the result because the token expiry check needs
|
|
them; keeping them here makes that dependency visible at the call site.
|
|
Private repos are included when a token is set.
|
|
"""
|
|
token = cfg["GITHUB_TOKEN"]
|
|
repos = []
|
|
headers = {}
|
|
page = 1
|
|
while True:
|
|
if token:
|
|
url = ("https://api.github.com/user/repos"
|
|
"?per_page=100&affiliation=owner&page=%d" % page)
|
|
else:
|
|
url = ("https://api.github.com/users/%s/repos"
|
|
"?per_page=100&type=owner&page=%d" % (cfg["GITHUB_USER"], page))
|
|
resp = api(url, token=token, scheme="Bearer")
|
|
headers = resp.headers
|
|
batch = resp.body
|
|
if not batch:
|
|
break
|
|
repos.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
|
|
if token:
|
|
# affiliation=owner still returns repos owned by orgs the user belongs to
|
|
repos = [r for r in repos
|
|
if r["owner"]["login"].lower() == cfg["GITHUB_USER"].lower()]
|
|
return repos, headers
|
|
|
|
|
|
def warn_token_expiry(headers):
|
|
"""Fine-grained PATs always expire; a silent expiry would stall the mirror.
|
|
|
|
GitHub reports the date in a response header, so every run can check it.
|
|
Returns the number of errors found, so an expired token can fail the run.
|
|
"""
|
|
raw = headers.get("github-authentication-token-expiration")
|
|
if not raw:
|
|
return 0 # classic token without expiry, or no token at all
|
|
text = raw.strip().replace(" UTC", "").replace("Z", "").replace("T", " ")
|
|
parsed = None
|
|
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d"):
|
|
try:
|
|
parsed = datetime.strptime(text.split(".")[0], fmt)
|
|
break
|
|
except ValueError:
|
|
continue
|
|
if parsed is None:
|
|
return 0
|
|
|
|
now_utc = datetime.now(timezone.utc).replace(tzinfo=None)
|
|
days = (parsed.replace(tzinfo=None) - now_utc).days
|
|
if days < 0:
|
|
log("ERROR", "GITHUB_TOKEN expired on %s - renew it" % parsed.date())
|
|
return 1
|
|
if days <= 14:
|
|
log("WARN", "GITHUB_TOKEN expires in %d day(s), on %s - renew it"
|
|
% (days, parsed.date()))
|
|
else:
|
|
log("INFO", "GITHUB_TOKEN valid until %s (%d days)" % (parsed.date(), days))
|
|
return 0
|
|
|
|
|
|
def duration_seconds(value):
|
|
"""Parse a Go-style duration ('24h', '8h0m0s') into seconds. -1 if unparseable."""
|
|
if not value:
|
|
return -1
|
|
total = 0
|
|
matched = False
|
|
for amount, unit in re.findall(r"(\d+)\s*([hms])", str(value)):
|
|
matched = True
|
|
total += int(amount) * {"h": 3600, "m": 60, "s": 1}[unit]
|
|
return total if matched else -1
|
|
|
|
|
|
def gitea_repo(cfg, name):
|
|
try:
|
|
return api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
|
|
token=cfg["GITEA_TOKEN"]).body
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
raise
|
|
|
|
|
|
def create_mirror(cfg, repo, dry_run):
|
|
name = repo["name"]
|
|
payload = {
|
|
"clone_addr": repo["clone_url"],
|
|
"repo_name": name,
|
|
"repo_owner": cfg["GITEA_OWNER"],
|
|
"mirror": True,
|
|
"mirror_interval": cfg["MIRROR_INTERVAL"],
|
|
"private": bool(repo.get("private")),
|
|
"description": (repo.get("description") or "")[:255],
|
|
"service": "github",
|
|
# Pull mirrors only ever sync git data, so importing the rest would be a
|
|
# one-off snapshot that silently goes stale. Keep it lean on the Pi.
|
|
"wiki": False,
|
|
"issues": False,
|
|
"labels": False,
|
|
"milestones": False,
|
|
"releases": False,
|
|
"pull_requests": False,
|
|
}
|
|
if cfg["GITHUB_TOKEN"]:
|
|
payload["auth_token"] = cfg["GITHUB_TOKEN"]
|
|
|
|
if dry_run:
|
|
log("DRY", "would mirror %s (private=%s)" % (name, payload["private"]))
|
|
return
|
|
|
|
api("%s/api/v1/repos/migrate" % cfg["GITEA_URL"], token=cfg["GITEA_TOKEN"],
|
|
method="POST", payload=payload)
|
|
log("OK", "mirrored %s (private=%s)" % (name, payload["private"]))
|
|
|
|
|
|
def align_interval(cfg, name, existing, dry_run):
|
|
wanted = duration_seconds(cfg["MIRROR_INTERVAL"])
|
|
current = duration_seconds(existing.get("mirror_interval"))
|
|
if wanted < 0 or current < 0 or wanted == current:
|
|
return
|
|
if dry_run:
|
|
log("DRY", "would set %s interval %s -> %s"
|
|
% (name, existing.get("mirror_interval"), cfg["MIRROR_INTERVAL"]))
|
|
return
|
|
api("%s/api/v1/repos/%s/%s" % (cfg["GITEA_URL"], cfg["GITEA_OWNER"], name),
|
|
token=cfg["GITEA_TOKEN"], method="PATCH",
|
|
payload={"mirror_interval": cfg["MIRROR_INTERVAL"]})
|
|
log("OK", "interval of %s set to %s" % (name, cfg["MIRROR_INTERVAL"]))
|
|
|
|
|
|
def github_releases(cfg, repo_name):
|
|
releases = []
|
|
page = 1
|
|
while True:
|
|
url = ("https://api.github.com/repos/%s/%s/releases?per_page=100&page=%d"
|
|
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), page))
|
|
batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer").body
|
|
if not batch:
|
|
break
|
|
releases.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
return releases
|
|
|
|
|
|
def github_release_assets(cfg, repo_name, release_id):
|
|
assets = []
|
|
page = 1
|
|
while True:
|
|
url = ("https://api.github.com/repos/%s/%s/releases/%s/assets"
|
|
"?per_page=100&page=%d"
|
|
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), release_id, page))
|
|
batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer").body
|
|
if not batch:
|
|
break
|
|
assets.extend(batch)
|
|
if len(batch) < 100:
|
|
break
|
|
page += 1
|
|
return assets
|
|
|
|
|
|
def gitea_releases(cfg, repo_name):
|
|
releases = []
|
|
page = 1
|
|
while True:
|
|
url = ("%s/api/v1/repos/%s/%s/releases?limit=50&page=%d"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), page))
|
|
batch = api(url, token=cfg["GITEA_TOKEN"]).body
|
|
if not batch:
|
|
break
|
|
releases.extend(batch)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
return releases
|
|
|
|
|
|
def gitea_tags(cfg, repo_name):
|
|
tags = set()
|
|
page = 1
|
|
while True:
|
|
url = ("%s/api/v1/repos/%s/%s/tags?limit=50&page=%d"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), page))
|
|
batch = api(url, token=cfg["GITEA_TOKEN"]).body
|
|
if not batch:
|
|
break
|
|
tags.update(tag["name"] for tag in batch)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
return tags
|
|
|
|
|
|
def release_payload(release):
|
|
return {
|
|
"tag_name": release["tag_name"],
|
|
"target_commitish": release.get("target_commitish") or "",
|
|
"name": release.get("name") or "",
|
|
"body": release.get("body") or "",
|
|
"draft": bool(release.get("draft")),
|
|
"prerelease": bool(release.get("prerelease")),
|
|
}
|
|
|
|
|
|
def release_needs_update(source, target):
|
|
"""Return true for fields Gitea's PATCH endpoint can actually change."""
|
|
wanted = release_payload(source)
|
|
if bool(target.get("draft")) != wanted["draft"]:
|
|
return True
|
|
if bool(target.get("prerelease")) != wanted["prerelease"]:
|
|
return True
|
|
# Gitea ignores empty strings in release PATCH requests. Retaining old text
|
|
# also follows this script's policy of never deleting backup information.
|
|
if wanted["name"] and (target.get("name") or "") != wanted["name"]:
|
|
return True
|
|
if wanted["body"] and (target.get("body") or "") != wanted["body"]:
|
|
return True
|
|
return False
|
|
|
|
|
|
def release_update_payload(source):
|
|
wanted = release_payload(source)
|
|
payload = {
|
|
"draft": wanted["draft"],
|
|
"prerelease": wanted["prerelease"],
|
|
}
|
|
for key in ("target_commitish", "name", "body"):
|
|
if wanted[key]:
|
|
payload[key] = wanted[key]
|
|
return payload
|
|
|
|
|
|
def sync_release_assets(cfg, repo_name, source_release, target_release, dry_run):
|
|
"""Return (copied, preserved, failed) counts for this release's assets."""
|
|
copied = preserved = failed = 0
|
|
source_assets = github_release_assets(cfg, repo_name, source_release["id"])
|
|
target_assets = {asset["name"]: asset for asset in target_release.get("assets", [])}
|
|
|
|
for asset in source_assets:
|
|
name = asset["name"]
|
|
existing = target_assets.get(name)
|
|
if existing is not None:
|
|
source_size = int(asset.get("size") or 0)
|
|
target_size = int(existing.get("size") or 0)
|
|
if source_size and target_size and source_size != target_size:
|
|
preserved += 1
|
|
log("WARN", "%s release %s asset %s differs (%d vs %d bytes) - "
|
|
"Gitea copy left untouched"
|
|
% (repo_name, source_release["tag_name"], name,
|
|
source_size, target_size))
|
|
continue
|
|
|
|
if dry_run:
|
|
log("DRY", "would copy %s release %s asset %s (%d bytes)"
|
|
% (repo_name, source_release["tag_name"], name,
|
|
int(asset.get("size") or 0)))
|
|
copied += 1
|
|
continue
|
|
|
|
tmp = None
|
|
try:
|
|
tmp, size = download_github_asset(cfg, asset)
|
|
upload_release_asset(cfg, repo_name, target_release["id"], asset, tmp, size)
|
|
copied += 1
|
|
log("OK", "copied %s release %s asset %s (%d bytes)"
|
|
% (repo_name, source_release["tag_name"], name, size))
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
log("ERROR", "%s release %s asset %s failed (HTTP %s): %s"
|
|
% (repo_name, source_release["tag_name"], name, exc.code, detail))
|
|
failed += 1
|
|
except Exception as exc:
|
|
log("ERROR", "%s release %s asset %s failed: %s"
|
|
% (repo_name, source_release["tag_name"], name, exc))
|
|
failed += 1
|
|
finally:
|
|
if tmp is not None:
|
|
tmp.close()
|
|
return copied, preserved, failed
|
|
|
|
|
|
def sync_releases(cfg, repo_name, dry_run):
|
|
stats = {"created": 0, "updated": 0, "assets": 0, "preserved": 0,
|
|
"waiting": 0, "errors": 0}
|
|
source_releases = github_releases(cfg, repo_name)
|
|
target_releases = gitea_releases(cfg, repo_name)
|
|
by_tag = {release["tag_name"]: release for release in target_releases}
|
|
tags_to_check = [
|
|
release["tag_name"] for release in source_releases
|
|
if not release.get("draft")
|
|
and (release["tag_name"] not in by_tag
|
|
or by_tag[release["tag_name"]].get("draft"))
|
|
]
|
|
available_tags = gitea_tags(cfg, repo_name) if tags_to_check else set()
|
|
|
|
source_tags = {release["tag_name"] for release in source_releases}
|
|
orphaned = sorted(tag for tag in by_tag if tag not in source_tags)
|
|
if orphaned:
|
|
log("WARN", "%s Gitea release(s) with no GitHub counterpart left untouched: %s"
|
|
% (repo_name, ", ".join(orphaned)))
|
|
|
|
for source in source_releases:
|
|
tag = source["tag_name"]
|
|
target = by_tag.get(tag)
|
|
needs_mirrored_tag = (not source.get("draft")
|
|
and (target is None or target.get("draft")))
|
|
if needs_mirrored_tag and tag not in available_tags:
|
|
stats["waiting"] += 1
|
|
log("WARN", "%s release %s waits for its tag to reach the Gitea mirror"
|
|
% (repo_name, tag))
|
|
continue
|
|
|
|
try:
|
|
if target is None:
|
|
if dry_run:
|
|
target = {"id": source["id"], "tag_name": tag, "assets": []}
|
|
log("DRY", "would create %s release %s" % (repo_name, tag))
|
|
else:
|
|
target = api(
|
|
"%s/api/v1/repos/%s/%s/releases"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name)),
|
|
token=cfg["GITEA_TOKEN"], method="POST",
|
|
payload=release_payload(source)).body
|
|
log("OK", "created %s release %s" % (repo_name, tag))
|
|
stats["created"] += 1
|
|
elif release_needs_update(source, target):
|
|
if dry_run:
|
|
log("DRY", "would update %s release %s" % (repo_name, tag))
|
|
else:
|
|
target = api(
|
|
"%s/api/v1/repos/%s/%s/releases/%s"
|
|
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
|
path_part(repo_name), target["id"]),
|
|
token=cfg["GITEA_TOKEN"], method="PATCH",
|
|
payload=release_update_payload(source)).body
|
|
log("OK", "updated %s release %s" % (repo_name, tag))
|
|
stats["updated"] += 1
|
|
|
|
if truthy(cfg["SYNC_RELEASE_ASSETS"]):
|
|
copied, preserved, failed = sync_release_assets(
|
|
cfg, repo_name, source, target, dry_run)
|
|
stats["assets"] += copied
|
|
stats["preserved"] += preserved
|
|
stats["errors"] += failed
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
log("ERROR", "%s release %s failed (HTTP %s): %s"
|
|
% (repo_name, tag, exc.code, detail))
|
|
stats["errors"] += 1
|
|
except Exception as exc:
|
|
log("ERROR", "%s release %s failed: %s" % (repo_name, tag, exc))
|
|
stats["errors"] += 1
|
|
return stats
|
|
|
|
|
|
def main():
|
|
cfg = load_config()
|
|
dry_run = truthy(cfg["DRY_RUN"])
|
|
|
|
missing = [k for k in ("GITHUB_USER", "GITEA_URL", "GITEA_TOKEN", "GITEA_OWNER")
|
|
if not cfg[k]]
|
|
if missing:
|
|
log("ERROR", "Missing config value(s): %s" % ", ".join(missing))
|
|
return 2
|
|
|
|
if not cfg["GITHUB_TOKEN"]:
|
|
log("WARN", "GITHUB_TOKEN is empty - private repositories will be skipped")
|
|
|
|
try:
|
|
repos, github_headers = github_repos(cfg)
|
|
except urllib.error.HTTPError as exc:
|
|
log("ERROR", "GitHub API %s: %s" % (exc.code, exc.read().decode("utf-8", "replace")[:300]))
|
|
return 2
|
|
except Exception as exc:
|
|
log("ERROR", "GitHub API unreachable: %s" % exc)
|
|
return 2
|
|
|
|
errors = warn_token_expiry(github_headers)
|
|
|
|
if not truthy(cfg["INCLUDE_FORKS"]):
|
|
repos = [r for r in repos if not r.get("fork")]
|
|
if not truthy(cfg["INCLUDE_ARCHIVED"]):
|
|
repos = [r for r in repos if not r.get("archived")]
|
|
|
|
private_count = sum(1 for r in repos if r.get("private"))
|
|
log("INFO", "GitHub: %d repositor%s to consider (%d public, %d private)%s"
|
|
% (len(repos), "y" if len(repos) == 1 else "ies",
|
|
len(repos) - private_count, private_count,
|
|
" (DRY RUN)" if dry_run else ""))
|
|
|
|
# A fine-grained token defaults to "Public repositories", which looks exactly
|
|
# like an account without private repos. Name the likely cause explicitly.
|
|
if cfg["GITHUB_TOKEN"] and private_count == 0:
|
|
log("WARN", "Token is set but GitHub reports no private repositories - "
|
|
"check that its repository access is 'All repositories', "
|
|
"not 'Public repositories'")
|
|
|
|
created = skipped = 0
|
|
release_totals = {"created": 0, "updated": 0, "assets": 0,
|
|
"preserved": 0, "waiting": 0}
|
|
for repo in sorted(repos, key=lambda r: r["name"].lower()):
|
|
name = repo["name"]
|
|
try:
|
|
existing = gitea_repo(cfg, name)
|
|
except Exception as exc:
|
|
log("ERROR", "%s: cannot query Gitea: %s" % (name, exc))
|
|
errors += 1
|
|
continue
|
|
|
|
if existing is None:
|
|
mirror_created = False
|
|
try:
|
|
create_mirror(cfg, repo, dry_run)
|
|
created += 1
|
|
mirror_created = True
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
log("ERROR", "%s: migrate failed (HTTP %s): %s" % (name, exc.code, detail))
|
|
errors += 1
|
|
except Exception as exc:
|
|
log("ERROR", "%s: migrate failed: %s" % (name, exc))
|
|
errors += 1
|
|
|
|
if mirror_created and truthy(cfg["SYNC_RELEASES"]):
|
|
try:
|
|
if dry_run:
|
|
count = len(github_releases(cfg, name))
|
|
if count:
|
|
log("DRY", "would synchronize %d release(s) after creating %s"
|
|
% (count, name))
|
|
else:
|
|
stats = sync_releases(cfg, name, dry_run)
|
|
for key in release_totals:
|
|
release_totals[key] += stats[key]
|
|
errors += stats["errors"]
|
|
except Exception as exc:
|
|
log("ERROR", "%s: could not synchronize releases: %s" % (name, exc))
|
|
errors += 1
|
|
continue
|
|
|
|
skipped += 1
|
|
if not existing.get("mirror"):
|
|
log("WARN", "%s exists in Gitea but is not a mirror - left untouched" % name)
|
|
continue
|
|
try:
|
|
align_interval(cfg, name, existing, dry_run)
|
|
except Exception as exc:
|
|
log("ERROR", "%s: could not update mirror interval: %s" % (name, exc))
|
|
errors += 1
|
|
|
|
if truthy(cfg["SYNC_RELEASES"]):
|
|
try:
|
|
stats = sync_releases(cfg, name, dry_run)
|
|
for key in release_totals:
|
|
release_totals[key] += stats[key]
|
|
errors += stats["errors"]
|
|
except urllib.error.HTTPError as exc:
|
|
detail = exc.read().decode("utf-8", "replace")[:300]
|
|
log("ERROR", "%s: release sync failed (HTTP %s): %s"
|
|
% (name, exc.code, detail))
|
|
errors += 1
|
|
except Exception as exc:
|
|
log("ERROR", "%s: release sync failed: %s" % (name, exc))
|
|
errors += 1
|
|
|
|
github_names = {r["name"].lower() for r in repos}
|
|
try:
|
|
page, orphans = 1, []
|
|
while True:
|
|
batch = api("%s/api/v1/user/repos?limit=50&page=%d" % (cfg["GITEA_URL"], page),
|
|
token=cfg["GITEA_TOKEN"]).body
|
|
if not batch:
|
|
break
|
|
orphans.extend(
|
|
r["name"] for r in batch
|
|
if r.get("mirror")
|
|
and r["owner"]["login"].lower() == cfg["GITEA_OWNER"].lower()
|
|
and r["name"].lower() not in github_names
|
|
)
|
|
if len(batch) < 50:
|
|
break
|
|
page += 1
|
|
if orphans:
|
|
log("WARN", "Mirrors with no GitHub counterpart (renamed or deleted upstream): %s"
|
|
% ", ".join(sorted(orphans)))
|
|
except Exception as exc:
|
|
log("WARN", "Could not check for orphaned mirrors: %s" % exc)
|
|
|
|
log("INFO", "Done: %d mirror(s) created, %d already present; releases: "
|
|
"%d created, %d updated, %d asset(s) copied, %d preserved, "
|
|
"%d waiting for tags; %d error(s)"
|
|
% (created, skipped, release_totals["created"], release_totals["updated"],
|
|
release_totals["assets"], release_totals["preserved"],
|
|
release_totals["waiting"], errors))
|
|
return 1 if errors else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|