diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..51f0497 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[registries.gitea] +index = "sparse+https://gt.hexor.cy/api/packages/Furumi/cargo/" +credential-provider = "cargo:token" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cece9b1..82130e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,8 @@ name: CI on: push: + branches: + - "**" pull_request: permissions: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 71d6ff1..9fd89d5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,20 +18,12 @@ env: CARGO_NET_RETRY: 10 jobs: - publish: + build: if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest timeout-minutes: 45 steps: - uses: actions/checkout@v4 - - name: Check registry credentials - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: | - if [ -z "$CARGO_REGISTRY_TOKEN" ]; then - echo "::error::Add the CARGO_REGISTRY_TOKEN repository Actions secret, then rerun this workflow." - exit 1 - fi - uses: dtolnay/rust-toolchain@stable with: toolchain: "1.97" @@ -42,7 +34,39 @@ jobs: - run: cargo check --workspace --all-targets --all-features --locked - run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - run: cargo test --workspace --locked - - name: Publish unpublished workspace versions + - name: Verify release packages once + run: python3 scripts/publish_crates.py --dry-run + + publish-crates-io: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97" + - name: Publish to crates.io without rebuilding env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - run: python3 scripts/publish_crates.py + run: python3 scripts/publish_crates.py --no-verify + + publish-gitea: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97" + - name: Publish mirror to Gitea without rebuilding + env: + GITEA_PACKAGES_TOKEN: ${{ secrets.GITEA_PACKAGES_TOKEN }} + run: | + if [ -z "$GITEA_PACKAGES_TOKEN" ]; then + echo "::error::Add the GITEA_PACKAGES_TOKEN repository Actions secret, then rerun this workflow." + exit 1 + fi + export CARGO_REGISTRIES_GITEA_TOKEN="Bearer $GITEA_PACKAGES_TOKEN" + python3 scripts/publish_crates.py --registry gitea --no-verify diff --git a/CHANGELOG.md b/CHANGELOG.md index 8196c14..849437b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,10 @@ Releases `music-dht 0.6.0` and `furumi-library 0.3.0`. `federation-net` remains at `0.3.1`. -- Tag-driven CI publishes every unpublished workspace crate version to - crates.io, in dependency order; already published versions are skipped on - retries. Authentication uses the `CARGO_REGISTRY_TOKEN` Actions secret. +- Tag-driven CI verifies release packages once, then publishes to crates.io + and the Furumi Gitea Cargo mirror in parallel jobs without recompiling. + Already published versions are skipped independently in each registry. + Authentication uses `CARGO_REGISTRY_TOKEN` and `GITEA_PACKAGES_TOKEN`. ### Podcast discovery (`podcast_dht`) diff --git a/README.md b/README.md index d453d6f..5e20f16 100644 --- a/README.md +++ b/README.md @@ -82,9 +82,14 @@ sandbox may need explicit network permission. ## Publishing -Add one repository Actions secret named `CARGO_REGISTRY_TOKEN`, containing a -crates.io API token with publish access to `federation-net`, `music-dht` and -`furumi-library`. No registry username or Gitea token secret is needed. +Add these repository Actions secrets: + +- `CARGO_REGISTRY_TOKEN`: a crates.io API token with publish access to + `federation-net`, `music-dht` and `furumi-library`. +- `GITEA_PACKAGES_TOKEN`: a raw Gitea personal access token with + `write:package` and permission to publish under the `Furumi` organization. + Create it under user Settings → Applications. The workflow adds the + required `Bearer` prefix itself; no username secret is needed. Update changed crate versions, workspace dependency requirements, `Cargo.lock` and `CHANGELOG.md`, then merge to `main`. Create `-v` tags for @@ -99,14 +104,20 @@ git push origin music-dht-v0.6.0 furumi-library-v0.3.0 v0.6.0 Only the `v*` workspace tag triggers `.github/workflows/publish.yml`; the crate tags identify each package release without starting duplicate uploads. -The workflow runs the workspace checks, selects every publishable workspace -crate whose exact version is absent from the crates.io sparse index, and lets -Cargo verify and publish the selected packages in dependency order. Versions -already published are skipped, so rerunning a partially completed release is -safe. A registry lookup error fails the release instead of assuming absence. +The workflow runs the workspace checks and verifies the release packages in +one build job. Two publishing jobs then upload without rebuilding +(`cargo publish --no-verify`), selecting missing versions independently in +crates.io and the Gitea mirror at +`https://gt.hexor.cy/api/packages/Furumi/cargo/`. The mirror includes all three +crates, including an unchanged version missing there. Already published +versions are skipped on retries; a registry lookup error fails the release +instead of assuming absence. Mirrored manifests retain crates.io dependency +references, so the mirror waits up to ten minutes for new workspace +dependencies to appear in crates.io before packaging its uploads. Preview the selection with `python3 scripts/publish_crates.py --plan`, or -verify the packages without uploading with `--dry-run`. If the secret was +verify the packages without uploading with `--dry-run`. Add `--registry gitea` +to preview or publish to the mirror. If a secret was missing or a publish failed, fix the cause and rerun the existing tag workflow; do not move release tags. Manual dispatch must select the release tag as its ref. Publication is performed by CI, not by the tagging commands above. diff --git a/scripts/publish_crates.py b/scripts/publish_crates.py index c8dee81..3678804 100644 --- a/scripts/publish_crates.py +++ b/scripts/publish_crates.py @@ -4,9 +4,15 @@ import argparse import json import os import subprocess +import time import urllib.error import urllib.request +INDEXES = { + "crates-io": "https://index.crates.io/", + "gitea": "https://gt.hexor.cy/api/packages/Furumi/cargo/", +} + def index_path(name): name = name.lower() @@ -17,9 +23,9 @@ def index_path(name): return f"{name[:2]}/{name[2:4]}/{name}" -def is_published(name, version): +def is_published(name, version, registry="crates-io"): request = urllib.request.Request( - f"https://index.crates.io/{index_path(name)}", + INDEXES[registry] + index_path(name), headers={"User-Agent": "frid-release-ci", "Cache-Control": "no-cache"}, ) try: @@ -32,17 +38,17 @@ def is_published(name, version): return any(row["vers"] == version for row in versions) -def unpublished_packages(metadata): +def unpublished_packages(metadata, registry="crates-io"): members = set(metadata["workspace_members"]) packages = [] for package in sorted(metadata["packages"], key=lambda item: item["name"]): registries = package["publish"] if package["id"] not in members or ( - registries is not None and "crates-io" not in registries + registries is not None and registry not in registries ): continue name, version = package["name"], package["version"] - if is_published(name, version): + if is_published(name, version, registry): print(f"Skip {name} {version}: already published", flush=True) else: print(f"Publish {name} {version}", flush=True) @@ -50,25 +56,54 @@ def unpublished_packages(metadata): return packages +def wait_for_mirror_dependencies(metadata, selected): + # Mirrored manifests retain their crates.io dependencies. In the parallel + # upload job, wait for new workspace dependencies to reach crates.io before + # Cargo resolves the mirror's packages (it does not compile them again). + members = {p["name"]: p for p in metadata["packages"] if p["id"] in metadata["workspace_members"]} + required = { + (dep["name"], members[dep["name"]]["version"]) + for name in selected + for dep in members[name]["dependencies"] + if dep.get("path") and dep["name"] in members and dep["kind"] != "dev" + } + deadline = time.monotonic() + 600 + while required: + required = {(name, version) for name, version in required if not is_published(name, version)} + if not required: + return + if time.monotonic() >= deadline: + raise RuntimeError(f"Workspace dependencies did not reach crates.io: {sorted(required)}") + print(f"Waiting for crates.io dependencies before mirroring: {sorted(required)}", flush=True) + time.sleep(10) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--plan", action="store_true", help="list packages without publishing") parser.add_argument("--dry-run", action="store_true", help="verify packages without uploading") + parser.add_argument("--registry", choices=INDEXES, default="crates-io") + parser.add_argument("--no-verify", action="store_true", help="publish packages already verified by CI without rebuilding") args = parser.parse_args() - if not (args.plan or args.dry_run) and not os.environ.get("CARGO_REGISTRY_TOKEN"): - parser.error("CARGO_REGISTRY_TOKEN must be set for publishing") + token_variable = "CARGO_REGISTRY_TOKEN" if args.registry == "crates-io" else "CARGO_REGISTRIES_GITEA_TOKEN" + if not (args.plan or args.dry_run) and not os.environ.get(token_variable): + parser.error(f"{token_variable} must be set for publishing") metadata = json.loads(subprocess.check_output( ["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"], text=True, )) - packages = unpublished_packages(metadata) + packages = unpublished_packages(metadata, args.registry) if args.plan or not packages: return - command = ["cargo", "publish", "--registry", "crates-io", "--locked", "--all-features"] + if args.registry == "gitea": + wait_for_mirror_dependencies(metadata, packages) + command = ["cargo", "publish", "--registry", args.registry, "--locked", "--all-features"] for name in packages: command.extend(["--package", name]) if args.dry_run: command.append("--dry-run") + if args.no_verify: + command.append("--no-verify") # Multi-package publishing verifies against a temporary local registry, # then uploads and waits for dependencies to appear before their users. subprocess.run(command, check=True) diff --git a/scripts/test_publish_crates.py b/scripts/test_publish_crates.py index 2557f95..66bcb77 100644 --- a/scripts/test_publish_crates.py +++ b/scripts/test_publish_crates.py @@ -1,4 +1,5 @@ import io +import json import unittest import urllib.error from contextlib import redirect_stdout @@ -8,6 +9,18 @@ import publish_crates class PublishCratesTests(unittest.TestCase): + def test_publish_job_does_not_rebuild_verified_packages(self): + with patch("sys.argv", ["publish_crates.py", "--no-verify"]): + with patch.dict(publish_crates.os.environ, {"CARGO_REGISTRY_TOKEN": "test-only"}): + with patch.object(publish_crates.subprocess, "check_output", return_value=json.dumps({})): + with patch.object(publish_crates, "unpublished_packages", return_value=["music-dht", "furumi-library"]): + with patch.object(publish_crates.subprocess, "run") as run: + publish_crates.main() + run.assert_called_once_with([ + "cargo", "publish", "--registry", "crates-io", "--locked", "--all-features", + "--package", "music-dht", "--package", "furumi-library", "--no-verify", + ], check=True) + def test_existing_and_private_versions_are_excluded_on_retry(self): packages = [ {"id": "net", "name": "federation-net", "version": "0.3.1", "publish": None}, @@ -23,9 +36,30 @@ class PublishCratesTests(unittest.TestCase): ({"federation-net", "music-dht"}, ["furumi-library"]), ]: with self.subTest(published=published), redirect_stdout(io.StringIO()): - with patch.object(publish_crates, "is_published", side_effect=lambda name, _: name in published): + with patch.object(publish_crates, "is_published", side_effect=lambda name, *args: name in published): self.assertEqual(publish_crates.unpublished_packages(metadata), expected) + def test_mirror_waits_for_workspace_dependencies_without_building(self): + metadata = {"workspace_members": ["lib", "dht"], "packages": [ + {"id": "lib", "name": "furumi-library", "version": "0.3.0", "dependencies": [ + {"name": "music-dht", "path": "/workspace/music-dht", "kind": None}, + ]}, + {"id": "dht", "name": "music-dht", "version": "0.6.0", "dependencies": []}, + ]} + with patch.object(publish_crates, "is_published", side_effect=[False, True]) as published: + with patch.object(publish_crates.time, "sleep") as sleep, redirect_stdout(io.StringIO()): + publish_crates.wait_for_mirror_dependencies(metadata, ["furumi-library"]) + self.assertEqual(published.call_count, 2) + sleep.assert_called_once_with(10) + + def test_gitea_uses_its_own_index(self): + response = MagicMock() + response.__enter__.return_value.read.return_value = b'{"vers":"0.3.1"}\n' + with patch.object(publish_crates.urllib.request, "urlopen", return_value=response) as fetch: + self.assertTrue(publish_crates.is_published("federation-net", "0.3.1", "gitea")) + self.assertEqual(fetch.call_args.args[0].full_url, + "https://gt.hexor.cy/api/packages/Furumi/cargo/fe/de/federation-net") + def test_registry_errors_do_not_look_like_unpublished_versions(self): for status in (403, 429, 500): error = urllib.error.HTTPError("https://index.crates.io/test", status, "error", {}, None)