Mirror releases to Gitea after one shared build
CI / check (push) Canceled after 1m16s

This commit is contained in:
Aleksandr Bogomiakov committed 2026-11-20 14:01:56 +00:00
1 parent 84c5eb571c
commit 8af4d2cbbc
7 files changed
+143 -33

No files matched your search

+44 -9
View File
@@ -4,9 +4,15 @@ import argparse
import json
import os
import subprocess
import time
import urllib.error
import urllib.request
INDEXES = {
"crates-io": "https://index.crates.io/",
"gitea": "https://gt.hexor.cy/api/packages/Furumi/cargo/",
}
def index_path(name):
name = name.lower()
@@ -17,9 +23,9 @@ def index_path(name):
return f"{name[:2]}/{name[2:4]}/{name}"
def is_published(name, version):
def is_published(name, version, registry="crates-io"):
request = urllib.request.Request(
f"https://index.crates.io/{index_path(name)}",
INDEXES[registry] + index_path(name),
headers={"User-Agent": "frid-release-ci", "Cache-Control": "no-cache"},
)
try:
@@ -32,17 +38,17 @@ def is_published(name, version):
return any(row["vers"] == version for row in versions)
def unpublished_packages(metadata):
def unpublished_packages(metadata, registry="crates-io"):
members = set(metadata["workspace_members"])
packages = []
for package in sorted(metadata["packages"], key=lambda item: item["name"]):
registries = package["publish"]
if package["id"] not in members or (
registries is not None and "crates-io" not in registries
registries is not None and registry not in registries
):
continue
name, version = package["name"], package["version"]
if is_published(name, version):
if is_published(name, version, registry):
print(f"Skip {name} {version}: already published", flush=True)
else:
print(f"Publish {name} {version}", flush=True)
@@ -50,25 +56,54 @@ def unpublished_packages(metadata):
return packages
def wait_for_mirror_dependencies(metadata, selected):
# Mirrored manifests retain their crates.io dependencies. In the parallel
# upload job, wait for new workspace dependencies to reach crates.io before
# Cargo resolves the mirror's packages (it does not compile them again).
members = {p["name"]: p for p in metadata["packages"] if p["id"] in metadata["workspace_members"]}
required = {
(dep["name"], members[dep["name"]]["version"])
for name in selected
for dep in members[name]["dependencies"]
if dep.get("path") and dep["name"] in members and dep["kind"] != "dev"
}
deadline = time.monotonic() + 600
while required:
required = {(name, version) for name, version in required if not is_published(name, version)}
if not required:
return
if time.monotonic() >= deadline:
raise RuntimeError(f"Workspace dependencies did not reach crates.io: {sorted(required)}")
print(f"Waiting for crates.io dependencies before mirroring: {sorted(required)}", flush=True)
time.sleep(10)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--plan", action="store_true", help="list packages without publishing")
parser.add_argument("--dry-run", action="store_true", help="verify packages without uploading")
parser.add_argument("--registry", choices=INDEXES, default="crates-io")
parser.add_argument("--no-verify", action="store_true", help="publish packages already verified by CI without rebuilding")
args = parser.parse_args()
if not (args.plan or args.dry_run) and not os.environ.get("CARGO_REGISTRY_TOKEN"):
parser.error("CARGO_REGISTRY_TOKEN must be set for publishing")
token_variable = "CARGO_REGISTRY_TOKEN" if args.registry == "crates-io" else "CARGO_REGISTRIES_GITEA_TOKEN"
if not (args.plan or args.dry_run) and not os.environ.get(token_variable):
parser.error(f"{token_variable} must be set for publishing")
metadata = json.loads(subprocess.check_output(
["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"],
text=True,
))
packages = unpublished_packages(metadata)
packages = unpublished_packages(metadata, args.registry)
if args.plan or not packages:
return
command = ["cargo", "publish", "--registry", "crates-io", "--locked", "--all-features"]
if args.registry == "gitea":
wait_for_mirror_dependencies(metadata, packages)
command = ["cargo", "publish", "--registry", args.registry, "--locked", "--all-features"]
for name in packages:
command.extend(["--package", name])
if args.dry_run:
command.append("--dry-run")
if args.no_verify:
command.append("--no-verify")
# Multi-package publishing verifies against a temporary local registry,
# then uploads and waits for dependencies to appear before their users.
subprocess.run(command, check=True)
+35 -1
View File
@@ -1,4 +1,5 @@
import io
import json
import unittest
import urllib.error
from contextlib import redirect_stdout
@@ -8,6 +9,18 @@ import publish_crates
class PublishCratesTests(unittest.TestCase):
def test_publish_job_does_not_rebuild_verified_packages(self):
with patch("sys.argv", ["publish_crates.py", "--no-verify"]):
with patch.dict(publish_crates.os.environ, {"CARGO_REGISTRY_TOKEN": "test-only"}):
with patch.object(publish_crates.subprocess, "check_output", return_value=json.dumps({})):
with patch.object(publish_crates, "unpublished_packages", return_value=["music-dht", "furumi-library"]):
with patch.object(publish_crates.subprocess, "run") as run:
publish_crates.main()
run.assert_called_once_with([
"cargo", "publish", "--registry", "crates-io", "--locked", "--all-features",
"--package", "music-dht", "--package", "furumi-library", "--no-verify",
], check=True)
def test_existing_and_private_versions_are_excluded_on_retry(self):
packages = [
{"id": "net", "name": "federation-net", "version": "0.3.1", "publish": None},
@@ -23,9 +36,30 @@ class PublishCratesTests(unittest.TestCase):
({"federation-net", "music-dht"}, ["furumi-library"]),
]:
with self.subTest(published=published), redirect_stdout(io.StringIO()):
with patch.object(publish_crates, "is_published", side_effect=lambda name, _: name in published):
with patch.object(publish_crates, "is_published", side_effect=lambda name, *args: name in published):
self.assertEqual(publish_crates.unpublished_packages(metadata), expected)
def test_mirror_waits_for_workspace_dependencies_without_building(self):
metadata = {"workspace_members": ["lib", "dht"], "packages": [
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "dependencies": [
{"name": "music-dht", "path": "/workspace/music-dht", "kind": None},
]},
{"id": "dht", "name": "music-dht", "version": "0.6.0", "dependencies": []},
]}
with patch.object(publish_crates, "is_published", side_effect=[False, True]) as published:
with patch.object(publish_crates.time, "sleep") as sleep, redirect_stdout(io.StringIO()):
publish_crates.wait_for_mirror_dependencies(metadata, ["furumi-library"])
self.assertEqual(published.call_count, 2)
sleep.assert_called_once_with(10)
def test_gitea_uses_its_own_index(self):
response = MagicMock()
response.__enter__.return_value.read.return_value = b'{"vers":"0.3.1"}\n'
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response) as fetch:
self.assertTrue(publish_crates.is_published("federation-net", "0.3.1", "gitea"))
self.assertEqual(fetch.call_args.args[0].full_url,
"https://gt.hexor.cy/api/packages/Furumi/cargo/fe/de/federation-net")
def test_registry_errors_do_not_look_like_unpublished_versions(self):
for status in (403, 429, 500):
error = urllib.error.HTTPError("https://index.crates.io/test", status, "error", {}, None)