diff --git a/README.md b/README.md index a13906b..236e8a2 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,11 @@ Keeping the repositories' *git content* up to date remains Gitea's job. | Pull new commits into **existing** mirrors | Gitea itself | `MIRROR_INTERVAL` | | Synchronize release metadata and uploaded assets | this script (cron) | daily | +### Architecture + +- [Architecture diagram](docs/architecture/sync_mirrors.architecture.html) +- [Sequence diagram](docs/architecture/sync_mirrors.sequence.html) + ### Requirements - Gitea 1.20+ with an API token (scopes `write:repository,read:user`) @@ -234,6 +239,11 @@ bleibt Giteas Aufgabe. | Commits in **bestehende** Mirrors holen | Gitea selbst | `MIRROR_INTERVAL` | | Release-Metadaten und hochgeladene Assets synchronisieren | dieses Skript (Cron) | täglich | +### Architektur + +- [Architecture-Diagramm](docs/architecture/sync_mirrors.architecture.html) +- [Sequence-Diagramm](docs/architecture/sync_mirrors.sequence.html) + ### Voraussetzungen - Gitea 1.20+ mit API-Token (Scopes `write:repository,read:user`) diff --git a/docs/architecture/sync_mirrors.architecture.html b/docs/architecture/sync_mirrors.architecture.html new file mode 100644 index 0000000..cfc2472 --- /dev/null +++ b/docs/architecture/sync_mirrors.architecture.html @@ -0,0 +1,13209 @@ + + + + + + + GitHub to Gitea Mirror Architecture Diagram + + + + + + + + + + + +
+ +
+
+
+

GitHub to Gitea Mirror Architecture

+
+

Runtime components and trust boundaries used by sync_mirrors.py for repository mirrors, releases, and streamed release assets

+
+ + + + + + +
+ + GitHub to Gitea Mirror Architecture + Runtime components and trust boundaries used by sync_mirrors.py for repository mirrors, releases, and streamed release assets + + + + + + + + + + + + + + + + + + + + + + + + + Local host running sync_mirrors.py + + + Configuration and credentials + + + GitHub account + + + Gitea instance + + + + + + + + + + + + + + + Runner · cron or shell · Local host running sync_mirrors.py + + + + Runner + cron or shell + + + + Config · env + tokens · Local host running sync_mirrors.py › Configuration and credentials + + + + Config + env + tokens + + + + Sync Script · coordinator · Local host running sync_mirrors.py + + + + Sync Script + coordinator + + + + Temp Asset · chunk buffer · Local host running sync_mirrors.py + + + + Temp Asset + chunk buffer + + + + GitHub API · REST · GitHub account · Bearer token + + + + GitHub API + REST + Bearer token + + + + Source Repos · owned projects · GitHub account + + + + Source Repos + owned projects + + + + GH Releases · metadata + assets · GitHub account + + + + GH Releases + metadata + assets + + + + Gitea API · local or LAN · Gitea instance · token + + + + Gitea API + local or LAN + token + + + + Pull Mirrors · git backup · Gitea instance + + + + Pull Mirrors + git backup + + + + Gitea Releases · metadata + assets · Gitea instance + + + + Gitea Releases + metadata + assets + + + + + + starts run + + + + loads settings + + + + lists repos + + + + reads + + + + checks mirrors + + + + create or patch + + + + streams asset + + + + uploads asset + + + + create or patch + + + + + +

+ + + + + + + + +
+ + +
+
+
+
+

Primary responsibility

+
+
    +
  • • GitHub is the source of repository and release inventory
    • +
  • • The script creates missing Gitea pull mirrors and aligns mirror intervals
    • +
  • • Existing Gitea content is preserved rather than deleted
    • +
+
+ +
+
+
+

Trust boundary

+
+
    +
  • • GitHub requests use a Bearer token when configured
    • +
  • • Gitea requests use token authentication
    • +
  • • Config values come from mirror.env and environment variables
    • +
+
+ +
+
+
+

Asset path

+
+
    +
  • • Release assets are downloaded in chunks
    • +
  • • A temporary file avoids buffering large assets in memory
    • +
  • • Multipart upload sends the same stream to Gitea
    • +
+
+
+ + +

+ Architecture diagram • Built with Archify • Create yours ↗ • Hover to trace • R route • Click to focus • +/− zoom • M radar • [/] views • P play story • T theme • E export +

+
+ + + + diff --git a/docs/architecture/sync_mirrors.architecture.json b/docs/architecture/sync_mirrors.architecture.json new file mode 100644 index 0000000..a9125be --- /dev/null +++ b/docs/architecture/sync_mirrors.architecture.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "diagram_type": "architecture", + "meta": { + "title": "GitHub to Gitea Mirror Architecture", + "subtitle": "Runtime components and trust boundaries used by sync_mirrors.py for repository mirrors, releases, and streamed release assets", + "output": "sync_mirrors.architecture.html", + "viewBox": [1180, 780], + "animation": "none", + "visual_preset": "blueprint", + "quality_profile": "showcase", + "repository": { + "url": "https://github.com/Andre-Bzk/github-to-gitea-mirror.git", + "revision": "18e926a764b6ee969ea61132bd446e22ab250a5b" + }, + "views": [ + { + "id": "local-runtime", + "label": "Local runtime", + "focus": ["runner", "config", "script", "tmp_asset"], + "note": "The scheduled or manual run loads configuration and keeps large assets out of RAM." + }, + { + "id": "repository-sync", + "label": "Repository mirror sync", + "focus": ["script", "github_api", "github_repos", "gitea_api", "gitea_mirrors"], + "note": "GitHub repository inventory drives Gitea mirror creation and interval alignment." + }, + { + "id": "release-assets", + "label": "Release and asset sync", + "focus": ["script", "github_releases", "tmp_asset", "gitea_releases"], + "note": "Release metadata is reconciled and missing assets stream through a temporary file." + } + ] + }, + "components": [ + { + "id": "runner", + "type": "external", + "label": "Runner", + "sublabel": "cron or shell", + "pos": [70, 355], + "size": [126, 62] + }, + { + "id": "config", + "type": "security", + "label": "Config", + "sublabel": "env + tokens", + "pos": [305, 155], + "size": [134, 64], + "sources": [ + { + "path": "sync_mirrors.py", + "label": "load_config" + } + ] + }, + { + "id": "script", + "type": "backend", + "label": "Sync Script", + "sublabel": "coordinator", + "pos": [300, 355], + "size": [144, 66], + "sources": [ + { + "path": "sync_mirrors.py", + "label": "main" + } + ] + }, + { + "id": "tmp_asset", + "type": "database", + "label": "Temp Asset", + "sublabel": "chunk buffer", + "pos": [565, 430], + "size": [144, 64], + "sources": [ + { + "path": "sync_mirrors.py", + "label": "download asset" + } + ] + }, + { + "id": "github_api", + "type": "cloud", + "label": "GitHub API", + "sublabel": "REST", + "pos": [565, 165], + "size": [136, 64], + "tag": "Bearer token" + }, + { + "id": "github_repos", + "type": "database", + "label": "Source Repos", + "sublabel": "owned projects", + "pos": [810, 95], + "size": [142, 64] + }, + { + "id": "github_releases", + "type": "database", + "label": "GH Releases", + "sublabel": "metadata + assets", + "pos": [810, 255], + "size": [148, 64] + }, + { + "id": "gitea_api", + "type": "cloud", + "label": "Gitea API", + "sublabel": "local or LAN", + "pos": [565, 610], + "size": [136, 64], + "tag": "token" + }, + { + "id": "gitea_mirrors", + "type": "database", + "label": "Pull Mirrors", + "sublabel": "git backup", + "pos": [810, 540], + "size": [142, 64] + }, + { + "id": "gitea_releases", + "type": "database", + "label": "Gitea Releases", + "sublabel": "metadata + assets", + "pos": [810, 680], + "size": [150, 64] + } + ], + "boundaries": [ + { + "kind": "region", + "label": "Local host running sync_mirrors.py", + "wraps": ["runner", "config", "script", "tmp_asset"] + }, + { + "kind": "security-group", + "label": "Configuration and credentials", + "wraps": ["config"] + }, + { + "kind": "region", + "label": "GitHub account", + "wraps": ["github_api", "github_repos", "github_releases"] + }, + { + "kind": "region", + "label": "Gitea instance", + "wraps": ["gitea_api", "gitea_mirrors", "gitea_releases"] + } + ], + "connections": [ + { + "id": "starts", + "from": "runner", + "to": "script", + "label": "starts run", + "variant": "emphasis", + "labelDy": 46 + }, + { + "id": "loads", + "from": "script", + "to": "config", + "label": "loads settings", + "variant": "security" + }, + { + "id": "lists-repos", + "from": "script", + "to": "github_api", + "label": "lists repos", + "variant": "emphasis" + }, + { + "id": "repo-source", + "from": "github_api", + "to": "github_repos", + "label": "reads", + "variant": "default" + }, + { + "id": "checks-mirror", + "from": "script", + "to": "gitea_api", + "label": "checks mirrors", + "variant": "emphasis" + }, + { + "id": "mirror-target", + "from": "gitea_api", + "to": "gitea_mirrors", + "label": "create or patch", + "variant": "emphasis" + }, + { + "id": "asset-download", + "from": "github_releases", + "to": "tmp_asset", + "label": "streams asset", + "variant": "dashed" + }, + { + "id": "asset-upload", + "from": "tmp_asset", + "to": "gitea_api", + "label": "uploads asset", + "variant": "dashed", + "fromSide": "bottom", + "toSide": "top" + }, + { + "id": "release-api", + "from": "gitea_api", + "to": "gitea_releases", + "label": "create or patch", + "variant": "default" + } + ], + "cards": [ + { + "dot": "cyan", + "title": "Primary responsibility", + "items": [ + "GitHub is the source of repository and release inventory", + "The script creates missing Gitea pull mirrors and aligns mirror intervals", + "Existing Gitea content is preserved rather than deleted" + ] + }, + { + "dot": "rose", + "title": "Trust boundary", + "items": [ + "GitHub requests use a Bearer token when configured", + "Gitea requests use token authentication", + "Config values come from mirror.env and environment variables" + ] + }, + { + "dot": "emerald", + "title": "Asset path", + "items": [ + "Release assets are downloaded in chunks", + "A temporary file avoids buffering large assets in memory", + "Multipart upload sends the same stream to Gitea" + ] + } + ] +} diff --git a/docs/architecture/sync_mirrors.sequence.html b/docs/architecture/sync_mirrors.sequence.html new file mode 100644 index 0000000..a50aeb2 --- /dev/null +++ b/docs/architecture/sync_mirrors.sequence.html @@ -0,0 +1,13317 @@ + + + + + + + GitHub to Gitea Mirror Sync Diagram + + + + + + + + + + + +
+ +
+
+
+

GitHub to Gitea Mirror Sync

+
+

Runtime sequence of sync_mirrors.py: config, repository mirroring, release sync, streamed asset copy, orphan check, and exit status

+
+ + + + + + +
+ + GitHub to Gitea Mirror Sync + Runtime sequence of sync_mirrors.py: config, repository mirroring, release sync, streamed asset copy, orphan check, and exit status + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + start sync run + + + + + + + + load defaults, mirror.env, environment + + + + + + + + normalized settings + + + + + + + + list owned repositories + + + + + + + + repos + token expiry header + + + + + + + + query matching repository + + + + + + + + existing mirror or 404 + + + + + + + + migrate missing mirror or patch interval + + + + + + + + mirror created or updated + + + + + + + + list repository releases + + + + + + + + source release metadata + + + + + + + + list releases and tags + + + + + + + + target releases, assets, tags + + + + + + + + create or patch changed release + + + + + + + + stream missing asset + + + + + + + + multipart upload + + + + + + + + list mirrors for orphan warning + + + + + + + + summary log and exit code + + + + + + + Startup + + + + Mirror loop + + + + Release sync + + + + Finish + + + + + Runner · cron or shell · Sequence participant + + + + Runner + cron or shell + + + + Sync Script · mirror coordinator · Sequence participant + + + + Sync Script + mirror coordinator + + + + Config · mirror.env + env · Sequence participant + + + + Config + mirror.env + env + + + + GitHub API · repos, releases · Sequence participant + + + + GitHub API + repos, releases + + + + Gitea API · mirrors and releases · Sequence participant + + + + Gitea API + mirrors and releases + + + + Temp File · streamed asset buffer · Sequence participant + + + + Temp File + streamed asset buffer + + + + + Legend + + + request + + + + return + + + + security + + + + async trace + + + + default message + + + +

+ + + + + + + + +
+ + +
+
+
+
+

Mirror ownership

+
+
    +
  • • GitHub repositories drive the run; Gitea state is reconciled per repository
    • +
  • • Existing non-mirror repositories are warned about and left untouched
    • +
  • • Deleted or renamed upstream repositories are reported as orphaned mirrors
    • +
+
+ +
+
+
+

Release policy

+
+
    +
  • • Release metadata is created or patched only when fields differ
    • +
  • • Draft handling waits until the mirrored tag exists in Gitea
    • +
  • • No Gitea-side release or asset is deleted by the script
    • +
+
+ +
+
+
+

Asset streaming

+
+
    +
  • • GitHub assets download in chunks into a temporary file
    • +
  • • Multipart upload streams the temporary file to Gitea
    • +
  • • Existing same-name assets are preserved, with size mismatch warnings
    • +
+
+
+ + +

+ Sequence diagram • Built with Archify • Create yours ↗ • Hover to trace • R route • Click to focus • +/− zoom • M radar • [/] views • P play story • T theme • E export +

+
+ + + + diff --git a/docs/architecture/sync_mirrors.sequence.json b/docs/architecture/sync_mirrors.sequence.json new file mode 100644 index 0000000..c1ff6d7 --- /dev/null +++ b/docs/architecture/sync_mirrors.sequence.json @@ -0,0 +1,318 @@ +{ + "schema_version": 1, + "diagram_type": "sequence", + "meta": { + "title": "GitHub to Gitea Mirror Sync", + "subtitle": "Runtime sequence of sync_mirrors.py: config, repository mirroring, release sync, streamed asset copy, orphan check, and exit status", + "output": "sync_mirrors.sequence.html", + "viewBox": [940, 1060], + "animation": "none", + "visual_preset": "signal-flow", + "quality_profile": "showcase", + "views": [ + { + "id": "startup", + "label": "Startup and repository list", + "focus": ["runner", "script", "config", "github"], + "note": "The script merges file and environment config, then lists owned GitHub repositories." + }, + { + "id": "repo-loop", + "label": "Repository mirror loop", + "focus": ["script", "github", "gitea"], + "note": "Each GitHub repository is matched against Gitea before migration or interval alignment." + }, + { + "id": "release-assets", + "label": "Release and asset sync", + "focus": ["github", "script", "tmpfile", "gitea"], + "note": "Release metadata is reconciled and missing assets stream through a temporary file." + } + ] + }, + "participants": [ + { + "id": "runner", + "type": "external", + "label": "Runner", + "sublabel": "cron or shell" + }, + { + "id": "script", + "type": "backend", + "label": "Sync Script", + "sublabel": "mirror coordinator" + }, + { + "id": "config", + "type": "security", + "label": "Config", + "sublabel": "mirror.env + env" + }, + { + "id": "github", + "type": "cloud", + "label": "GitHub API", + "sublabel": "repos, releases" + }, + { + "id": "gitea", + "type": "cloud", + "label": "Gitea API", + "sublabel": "mirrors and releases" + }, + { + "id": "tmpfile", + "type": "database", + "label": "Temp File", + "sublabel": "streamed asset buffer" + } + ], + "segments": [ + { + "from": 150, + "to": 318, + "label": "Startup" + }, + { + "from": 334, + "to": 578, + "label": "Mirror loop" + }, + { + "from": 596, + "to": 844, + "label": "Release sync" + }, + { + "from": 862, + "to": 960, + "label": "Finish" + } + ], + "messages": [ + { + "id": "run-script", + "from": "runner", + "to": "script", + "y": 180, + "label": "start sync run", + "variant": "emphasis" + }, + { + "id": "load-config", + "from": "script", + "to": "config", + "y": 222, + "label": "load defaults, mirror.env, environment", + "variant": "security" + }, + { + "id": "config-values", + "from": "config", + "to": "script", + "y": 264, + "label": "normalized settings", + "variant": "return" + }, + { + "id": "list-repos", + "from": "script", + "to": "github", + "y": 306, + "label": "list owned repositories", + "variant": "emphasis" + }, + { + "id": "repos-back", + "from": "github", + "to": "script", + "y": 348, + "label": "repos + token expiry header", + "variant": "return" + }, + { + "id": "check-repo", + "from": "script", + "to": "gitea", + "y": 390, + "label": "query matching repository", + "variant": "default" + }, + { + "id": "repo-state", + "from": "gitea", + "to": "script", + "y": 432, + "label": "existing mirror or 404", + "variant": "return" + }, + { + "id": "create-or-align", + "from": "script", + "to": "gitea", + "y": 474, + "label": "migrate missing mirror or patch interval", + "variant": "emphasis" + }, + { + "id": "mirror-result", + "from": "gitea", + "to": "script", + "y": 516, + "label": "mirror created or updated", + "variant": "return" + }, + { + "id": "get-source-releases", + "from": "script", + "to": "github", + "y": 618, + "label": "list repository releases", + "variant": "default" + }, + { + "id": "source-releases", + "from": "github", + "to": "script", + "y": 660, + "label": "source release metadata", + "variant": "return" + }, + { + "id": "get-target-releases", + "from": "script", + "to": "gitea", + "y": 702, + "label": "list releases and tags", + "variant": "default" + }, + { + "id": "target-state", + "from": "gitea", + "to": "script", + "y": 744, + "label": "target releases, assets, tags", + "variant": "return" + }, + { + "id": "upsert-release", + "from": "script", + "to": "gitea", + "y": 786, + "label": "create or patch changed release", + "variant": "emphasis" + }, + { + "id": "download-asset", + "from": "github", + "to": "tmpfile", + "y": 828, + "label": "stream missing asset", + "variant": "dashed" + }, + { + "id": "upload-asset", + "from": "tmpfile", + "to": "gitea", + "y": 870, + "label": "multipart upload", + "variant": "dashed" + }, + { + "id": "orphan-check", + "from": "script", + "to": "gitea", + "y": 912, + "label": "list mirrors for orphan warning", + "variant": "default" + }, + { + "id": "exit-code", + "from": "script", + "to": "runner", + "y": 954, + "label": "summary log and exit code", + "variant": "return" + } + ], + "activations": [ + { + "participant": "script", + "from": 174, + "to": 960, + "type": "backend" + }, + { + "participant": "config", + "from": 216, + "to": 270, + "type": "security" + }, + { + "participant": "github", + "from": 300, + "to": 354, + "type": "cloud" + }, + { + "participant": "gitea", + "from": 384, + "to": 522, + "type": "cloud" + }, + { + "participant": "github", + "from": 612, + "to": 666, + "type": "cloud" + }, + { + "participant": "gitea", + "from": 696, + "to": 792, + "type": "cloud" + }, + { + "participant": "tmpfile", + "from": 822, + "to": 876, + "type": "database" + }, + { + "participant": "gitea", + "from": 864, + "to": 918, + "type": "cloud" + } + ], + "cards": [ + { + "dot": "cyan", + "title": "Mirror ownership", + "items": [ + "GitHub repositories drive the run; Gitea state is reconciled per repository", + "Existing non-mirror repositories are warned about and left untouched", + "Deleted or renamed upstream repositories are reported as orphaned mirrors" + ] + }, + { + "dot": "orange", + "title": "Release policy", + "items": [ + "Release metadata is created or patched only when fields differ", + "Draft handling waits until the mirrored tag exists in Gitea", + "No Gitea-side release or asset is deleted by the script" + ] + }, + { + "dot": "emerald", + "title": "Asset streaming", + "items": [ + "GitHub assets download in chunks into a temporary file", + "Multipart upload streams the temporary file to Gitea", + "Existing same-name assets are preserved, with size mismatch warnings" + ] + } + ] +}