mirror of
https://github.com/Andre-Bzk/github-to-gitea-mirror.git
synced 2026-11-15 09:33:13 +01:00
feat: synchronize GitHub releases with Gitea
Gitea pull mirrors preserve Git refs but leave release notes and uploaded artifacts outside the backup. Without a separate synchronization path, restoring a repository would still depend on GitHub for important release context and binaries.
This commit is contained in:
Andre
+375
-5
@@ -1,18 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mirror all GitHub repositories of a user into Gitea as pull mirrors.
|
||||
|
||||
Existing mirrors are left to Gitea's own cron; this script only creates the
|
||||
ones that are missing and keeps the mirror interval aligned with the config.
|
||||
It never deletes anything on the Gitea side -- a mirror that vanished upstream
|
||||
is exactly what a backup is supposed to keep.
|
||||
Existing git mirrors are left to Gitea's own cron. This script creates the ones
|
||||
that are missing, aligns their interval, and synchronizes GitHub release
|
||||
metadata and assets because Gitea's native mirror does not. It never deletes
|
||||
anything on the Gitea side -- retained history is the point of a backup.
|
||||
"""
|
||||
|
||||
import json
|
||||
import http.client
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
KNOWN_KEYS = (
|
||||
@@ -24,6 +29,8 @@ KNOWN_KEYS = (
|
||||
"MIRROR_INTERVAL",
|
||||
"INCLUDE_FORKS",
|
||||
"INCLUDE_ARCHIVED",
|
||||
"SYNC_RELEASES",
|
||||
"SYNC_RELEASE_ASSETS",
|
||||
"DRY_RUN",
|
||||
)
|
||||
|
||||
@@ -54,6 +61,8 @@ def load_config():
|
||||
"MIRROR_INTERVAL": "24h",
|
||||
"INCLUDE_FORKS": "false",
|
||||
"INCLUDE_ARCHIVED": "true",
|
||||
"SYNC_RELEASES": "true",
|
||||
"SYNC_RELEASE_ASSETS": "true",
|
||||
"DRY_RUN": "false",
|
||||
}
|
||||
if os.path.exists(CONFIG_PATH):
|
||||
@@ -102,6 +111,109 @@ def api(url, token=None, scheme="token", method="GET", payload=None):
|
||||
return resp.status, (json.loads(body) if body.strip() else None)
|
||||
|
||||
|
||||
def path_part(value):
|
||||
"""Quote one URL path component, including slashes used inside tag names."""
|
||||
return urllib.parse.quote(str(value), safe="")
|
||||
|
||||
|
||||
class SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
"""Do not forward API credentials to a cross-host download redirect."""
|
||||
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
|
||||
if redirected is None:
|
||||
return None
|
||||
old_host = urllib.parse.urlsplit(req.full_url).netloc.lower()
|
||||
new_host = urllib.parse.urlsplit(newurl).netloc.lower()
|
||||
if old_host != new_host:
|
||||
redirected.remove_header("Authorization")
|
||||
return redirected
|
||||
|
||||
|
||||
safe_download_opener = urllib.request.build_opener(SafeRedirectHandler())
|
||||
|
||||
|
||||
def download_github_asset(cfg, asset):
|
||||
"""Download a release asset into a temporary file without buffering it in RAM."""
|
||||
headers = {
|
||||
"Accept": "application/octet-stream",
|
||||
"User-Agent": "gitea-github-mirror/1.0",
|
||||
}
|
||||
if cfg["GITHUB_TOKEN"]:
|
||||
headers["Authorization"] = "Bearer %s" % cfg["GITHUB_TOKEN"]
|
||||
req = urllib.request.Request(asset["url"], headers=headers)
|
||||
tmp = tempfile.TemporaryFile()
|
||||
size = 0
|
||||
try:
|
||||
with safe_download_opener.open(req, timeout=180) as resp:
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
tmp.write(chunk)
|
||||
size += len(chunk)
|
||||
expected = int(asset.get("size") or 0)
|
||||
if expected and size != expected:
|
||||
raise IOError("downloaded %d bytes, expected %d" % (size, expected))
|
||||
tmp.seek(0)
|
||||
return tmp, size
|
||||
except Exception:
|
||||
tmp.close()
|
||||
raise
|
||||
|
||||
|
||||
def upload_release_asset(cfg, repo_name, release_id, asset, source, size):
|
||||
"""Stream a temporary file to Gitea as multipart/form-data."""
|
||||
query = urllib.parse.urlencode({"name": asset["name"]})
|
||||
url = ("%s/api/v1/repos/%s/%s/releases/%s/assets?%s"
|
||||
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]), path_part(repo_name),
|
||||
release_id, query))
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
boundary = "----gitea-github-mirror-%s" % uuid.uuid4().hex
|
||||
safe_name = str(asset["name"]).replace("\\", "_").replace('"', "_")
|
||||
safe_name = safe_name.replace("\r", "_").replace("\n", "_")
|
||||
content_type = asset.get("content_type") or "application/octet-stream"
|
||||
content_type = str(content_type).replace("\r", "").replace("\n", "")
|
||||
preamble = (
|
||||
"--%s\r\n"
|
||||
"Content-Disposition: form-data; name=\"attachment\"; filename=\"%s\"\r\n"
|
||||
"Content-Type: %s\r\n\r\n" % (boundary, safe_name, content_type)
|
||||
).encode("utf-8")
|
||||
ending = ("\r\n--%s--\r\n" % boundary).encode("ascii")
|
||||
request_path = parsed.path + (("?" + parsed.query) if parsed.query else "")
|
||||
|
||||
if parsed.scheme == "https":
|
||||
conn = http.client.HTTPSConnection(parsed.hostname, parsed.port, timeout=180)
|
||||
elif parsed.scheme == "http":
|
||||
conn = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=180)
|
||||
else:
|
||||
raise ValueError("Unsupported Gitea URL scheme: %s" % parsed.scheme)
|
||||
|
||||
try:
|
||||
conn.putrequest("POST", request_path)
|
||||
conn.putheader("Authorization", "token %s" % cfg["GITEA_TOKEN"])
|
||||
conn.putheader("Accept", "application/json")
|
||||
conn.putheader("Content-Type", "multipart/form-data; boundary=%s" % boundary)
|
||||
conn.putheader("Content-Length", str(len(preamble) + size + len(ending)))
|
||||
conn.putheader("User-Agent", "gitea-github-mirror/1.0")
|
||||
conn.endheaders()
|
||||
conn.send(preamble)
|
||||
while True:
|
||||
chunk = source.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
conn.send(chunk)
|
||||
conn.send(ending)
|
||||
response = conn.getresponse()
|
||||
body = response.read()
|
||||
if response.status >= 400:
|
||||
raise urllib.error.HTTPError(
|
||||
url, response.status, response.reason, response.headers, io.BytesIO(body))
|
||||
return json.loads(body.decode("utf-8")) if body.strip() else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def github_repos(cfg):
|
||||
"""All repos owned by the user; private ones included when a token is set."""
|
||||
token = cfg["GITHUB_TOKEN"]
|
||||
@@ -229,6 +341,229 @@ def align_interval(cfg, name, existing, dry_run):
|
||||
log("OK", "interval of %s set to %s" % (name, cfg["MIRROR_INTERVAL"]))
|
||||
|
||||
|
||||
def github_releases(cfg, repo_name):
|
||||
releases = []
|
||||
page = 1
|
||||
while True:
|
||||
url = ("https://api.github.com/repos/%s/%s/releases?per_page=100&page=%d"
|
||||
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), page))
|
||||
_, batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer")
|
||||
if not batch:
|
||||
break
|
||||
releases.extend(batch)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
return releases
|
||||
|
||||
|
||||
def github_release_assets(cfg, repo_name, release_id):
|
||||
assets = []
|
||||
page = 1
|
||||
while True:
|
||||
url = ("https://api.github.com/repos/%s/%s/releases/%s/assets"
|
||||
"?per_page=100&page=%d"
|
||||
% (path_part(cfg["GITHUB_USER"]), path_part(repo_name), release_id, page))
|
||||
_, batch = api(url, token=cfg["GITHUB_TOKEN"], scheme="Bearer")
|
||||
if not batch:
|
||||
break
|
||||
assets.extend(batch)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
page += 1
|
||||
return assets
|
||||
|
||||
|
||||
def gitea_releases(cfg, repo_name):
|
||||
releases = []
|
||||
page = 1
|
||||
while True:
|
||||
url = ("%s/api/v1/repos/%s/%s/releases?limit=50&page=%d"
|
||||
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
||||
path_part(repo_name), page))
|
||||
_, batch = api(url, token=cfg["GITEA_TOKEN"])
|
||||
if not batch:
|
||||
break
|
||||
releases.extend(batch)
|
||||
if len(batch) < 50:
|
||||
break
|
||||
page += 1
|
||||
return releases
|
||||
|
||||
|
||||
def gitea_tags(cfg, repo_name):
|
||||
tags = set()
|
||||
page = 1
|
||||
while True:
|
||||
url = ("%s/api/v1/repos/%s/%s/tags?limit=50&page=%d"
|
||||
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
||||
path_part(repo_name), page))
|
||||
_, batch = api(url, token=cfg["GITEA_TOKEN"])
|
||||
if not batch:
|
||||
break
|
||||
tags.update(tag["name"] for tag in batch)
|
||||
if len(batch) < 50:
|
||||
break
|
||||
page += 1
|
||||
return tags
|
||||
|
||||
|
||||
def release_payload(release):
|
||||
return {
|
||||
"tag_name": release["tag_name"],
|
||||
"target_commitish": release.get("target_commitish") or "",
|
||||
"name": release.get("name") or "",
|
||||
"body": release.get("body") or "",
|
||||
"draft": bool(release.get("draft")),
|
||||
"prerelease": bool(release.get("prerelease")),
|
||||
}
|
||||
|
||||
|
||||
def release_needs_update(source, target):
|
||||
"""Return true for fields Gitea's PATCH endpoint can actually change."""
|
||||
wanted = release_payload(source)
|
||||
if bool(target.get("draft")) != wanted["draft"]:
|
||||
return True
|
||||
if bool(target.get("prerelease")) != wanted["prerelease"]:
|
||||
return True
|
||||
# Gitea ignores empty strings in release PATCH requests. Retaining old text
|
||||
# also follows this script's policy of never deleting backup information.
|
||||
if wanted["name"] and (target.get("name") or "") != wanted["name"]:
|
||||
return True
|
||||
if wanted["body"] and (target.get("body") or "") != wanted["body"]:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def release_update_payload(source):
|
||||
wanted = release_payload(source)
|
||||
payload = {
|
||||
"draft": wanted["draft"],
|
||||
"prerelease": wanted["prerelease"],
|
||||
}
|
||||
for key in ("target_commitish", "name", "body"):
|
||||
if wanted[key]:
|
||||
payload[key] = wanted[key]
|
||||
return payload
|
||||
|
||||
|
||||
def sync_release_assets(cfg, repo_name, source_release, target_release, dry_run):
|
||||
copied = preserved = 0
|
||||
source_assets = github_release_assets(cfg, repo_name, source_release["id"])
|
||||
target_assets = {asset["name"]: asset for asset in target_release.get("assets", [])}
|
||||
|
||||
for asset in source_assets:
|
||||
name = asset["name"]
|
||||
existing = target_assets.get(name)
|
||||
if existing is not None:
|
||||
source_size = int(asset.get("size") or 0)
|
||||
target_size = int(existing.get("size") or 0)
|
||||
if source_size and target_size and source_size != target_size:
|
||||
preserved += 1
|
||||
log("WARN", "%s release %s asset %s differs (%d vs %d bytes) - "
|
||||
"Gitea copy left untouched"
|
||||
% (repo_name, source_release["tag_name"], name,
|
||||
source_size, target_size))
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
log("DRY", "would copy %s release %s asset %s (%d bytes)"
|
||||
% (repo_name, source_release["tag_name"], name,
|
||||
int(asset.get("size") or 0)))
|
||||
copied += 1
|
||||
continue
|
||||
|
||||
tmp = None
|
||||
try:
|
||||
tmp, size = download_github_asset(cfg, asset)
|
||||
upload_release_asset(cfg, repo_name, target_release["id"], asset, tmp, size)
|
||||
copied += 1
|
||||
log("OK", "copied %s release %s asset %s (%d bytes)"
|
||||
% (repo_name, source_release["tag_name"], name, size))
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
fail("%s release %s asset %s failed (HTTP %s): %s"
|
||||
% (repo_name, source_release["tag_name"], name, exc.code, detail))
|
||||
except Exception as exc:
|
||||
fail("%s release %s asset %s failed: %s"
|
||||
% (repo_name, source_release["tag_name"], name, exc))
|
||||
finally:
|
||||
if tmp is not None:
|
||||
tmp.close()
|
||||
return copied, preserved
|
||||
|
||||
|
||||
def sync_releases(cfg, repo_name, dry_run):
|
||||
stats = {"created": 0, "updated": 0, "assets": 0, "preserved": 0, "waiting": 0}
|
||||
source_releases = github_releases(cfg, repo_name)
|
||||
target_releases = gitea_releases(cfg, repo_name)
|
||||
by_tag = {release["tag_name"]: release for release in target_releases}
|
||||
tags_to_check = [
|
||||
release["tag_name"] for release in source_releases
|
||||
if not release.get("draft")
|
||||
and (release["tag_name"] not in by_tag
|
||||
or by_tag[release["tag_name"]].get("draft"))
|
||||
]
|
||||
available_tags = gitea_tags(cfg, repo_name) if tags_to_check else set()
|
||||
|
||||
source_tags = {release["tag_name"] for release in source_releases}
|
||||
orphaned = sorted(tag for tag in by_tag if tag not in source_tags)
|
||||
if orphaned:
|
||||
log("WARN", "%s Gitea release(s) with no GitHub counterpart left untouched: %s"
|
||||
% (repo_name, ", ".join(orphaned)))
|
||||
|
||||
for source in source_releases:
|
||||
tag = source["tag_name"]
|
||||
target = by_tag.get(tag)
|
||||
needs_mirrored_tag = (not source.get("draft")
|
||||
and (target is None or target.get("draft")))
|
||||
if needs_mirrored_tag and tag not in available_tags:
|
||||
stats["waiting"] += 1
|
||||
log("WARN", "%s release %s waits for its tag to reach the Gitea mirror"
|
||||
% (repo_name, tag))
|
||||
continue
|
||||
|
||||
try:
|
||||
if target is None:
|
||||
if dry_run:
|
||||
target = {"id": source["id"], "tag_name": tag, "assets": []}
|
||||
log("DRY", "would create %s release %s" % (repo_name, tag))
|
||||
else:
|
||||
_, target = api(
|
||||
"%s/api/v1/repos/%s/%s/releases"
|
||||
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
||||
path_part(repo_name)),
|
||||
token=cfg["GITEA_TOKEN"], method="POST",
|
||||
payload=release_payload(source))
|
||||
log("OK", "created %s release %s" % (repo_name, tag))
|
||||
stats["created"] += 1
|
||||
elif release_needs_update(source, target):
|
||||
if dry_run:
|
||||
log("DRY", "would update %s release %s" % (repo_name, tag))
|
||||
else:
|
||||
_, target = api(
|
||||
"%s/api/v1/repos/%s/%s/releases/%s"
|
||||
% (cfg["GITEA_URL"], path_part(cfg["GITEA_OWNER"]),
|
||||
path_part(repo_name), target["id"]),
|
||||
token=cfg["GITEA_TOKEN"], method="PATCH",
|
||||
payload=release_update_payload(source))
|
||||
log("OK", "updated %s release %s" % (repo_name, tag))
|
||||
stats["updated"] += 1
|
||||
|
||||
if truthy(cfg["SYNC_RELEASE_ASSETS"]):
|
||||
copied, preserved = sync_release_assets(
|
||||
cfg, repo_name, source, target, dry_run)
|
||||
stats["assets"] += copied
|
||||
stats["preserved"] += preserved
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
fail("%s release %s failed (HTTP %s): %s"
|
||||
% (repo_name, tag, exc.code, detail))
|
||||
except Exception as exc:
|
||||
fail("%s release %s failed: %s" % (repo_name, tag, exc))
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
cfg = load_config()
|
||||
dry_run = truthy(cfg["DRY_RUN"])
|
||||
@@ -272,6 +607,8 @@ def main():
|
||||
"not 'Public repositories'")
|
||||
|
||||
created = skipped = 0
|
||||
release_totals = {"created": 0, "updated": 0, "assets": 0,
|
||||
"preserved": 0, "waiting": 0}
|
||||
for repo in sorted(repos, key=lambda r: r["name"].lower()):
|
||||
name = repo["name"]
|
||||
try:
|
||||
@@ -281,14 +618,30 @@ def main():
|
||||
continue
|
||||
|
||||
if existing is None:
|
||||
mirror_created = False
|
||||
try:
|
||||
create_mirror(cfg, repo, dry_run)
|
||||
created += 1
|
||||
mirror_created = True
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
fail("%s: migrate failed (HTTP %s): %s" % (name, exc.code, detail))
|
||||
except Exception as exc:
|
||||
fail("%s: migrate failed: %s" % (name, exc))
|
||||
|
||||
if mirror_created and truthy(cfg["SYNC_RELEASES"]):
|
||||
try:
|
||||
if dry_run:
|
||||
count = len(github_releases(cfg, name))
|
||||
if count:
|
||||
log("DRY", "would synchronize %d release(s) after creating %s"
|
||||
% (count, name))
|
||||
else:
|
||||
stats = sync_releases(cfg, name, dry_run)
|
||||
for key in release_totals:
|
||||
release_totals[key] += stats[key]
|
||||
except Exception as exc:
|
||||
fail("%s: could not synchronize releases: %s" % (name, exc))
|
||||
continue
|
||||
|
||||
skipped += 1
|
||||
@@ -300,6 +653,18 @@ def main():
|
||||
except Exception as exc:
|
||||
fail("%s: could not update mirror interval: %s" % (name, exc))
|
||||
|
||||
if truthy(cfg["SYNC_RELEASES"]):
|
||||
try:
|
||||
stats = sync_releases(cfg, name, dry_run)
|
||||
for key in release_totals:
|
||||
release_totals[key] += stats[key]
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", "replace")[:300]
|
||||
fail("%s: release sync failed (HTTP %s): %s"
|
||||
% (name, exc.code, detail))
|
||||
except Exception as exc:
|
||||
fail("%s: release sync failed: %s" % (name, exc))
|
||||
|
||||
github_names = {r["name"].lower() for r in repos}
|
||||
try:
|
||||
page, orphans = 1, []
|
||||
@@ -323,7 +688,12 @@ def main():
|
||||
except Exception as exc:
|
||||
log("WARN", "Could not check for orphaned mirrors: %s" % exc)
|
||||
|
||||
log("INFO", "Done: %d created, %d already present, %d error(s)" % (created, skipped, errors))
|
||||
log("INFO", "Done: %d mirror(s) created, %d already present; releases: "
|
||||
"%d created, %d updated, %d asset(s) copied, %d preserved, "
|
||||
"%d waiting for tags; %d error(s)"
|
||||
% (created, skipped, release_totals["created"], release_totals["updated"],
|
||||
release_totals["assets"], release_totals["preserved"],
|
||||
release_totals["waiting"], errors))
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user