Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e47348b21d | ||
|
|
8af4d2cbbc | ||
|
|
84c5eb571c | ||
|
|
987063d616 | ||
|
|
3046c3f924 | ||
|
|
ddea053e9d |
No files matched your search
@@ -0,0 +1,3 @@
|
||||
[registries.gitea]
|
||||
index = "sparse+https://gt.hexor.cy/api/packages/Furumi/cargo/"
|
||||
credential-provider = "cargo:token"
|
||||
@@ -2,6 +2,8 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "**"
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
@@ -20,6 +22,7 @@ jobs:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features
|
||||
- run: cargo clippy --workspace --all-targets --all-features -- -D warnings
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
name: Publish crates
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v[0-9]*"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: crates-io-publish
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
CARGO_NET_RETRY: 10
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- run: python3 -m unittest discover -s scripts -p 'test_*.py'
|
||||
- run: cargo fmt --all -- --check
|
||||
- run: cargo check --workspace --all-targets --all-features --locked
|
||||
- run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
|
||||
- run: cargo test --workspace --locked
|
||||
- name: Verify release packages once
|
||||
run: python3 scripts/publish_crates.py --dry-run
|
||||
|
||||
publish-crates-io:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish to crates.io without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
run: python3 scripts/publish_crates.py --no-verify
|
||||
|
||||
publish-gitea:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: "1.97"
|
||||
- name: Publish mirror to Gitea without rebuilding
|
||||
env:
|
||||
CARGO_REGISTRIES_GITEA_TOKEN: ${{ secrets.CARGO_REGISTRIES_GITEA_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$CARGO_REGISTRIES_GITEA_TOKEN" ]; then
|
||||
echo "::error::Add the CARGO_REGISTRIES_GITEA_TOKEN repository Actions secret, then rerun this workflow."
|
||||
exit 1
|
||||
fi
|
||||
export CARGO_REGISTRIES_GITEA_TOKEN="Bearer $CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
python3 scripts/publish_crates.py --registry gitea --no-verify
|
||||
+12
-2
@@ -1,6 +1,16 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased — podcast discovery (`podcast_dht`)
|
||||
## v0.6.0 — 2026-11-20
|
||||
|
||||
Releases `music-dht 0.6.0` and `furumi-library 0.3.0`.
|
||||
`federation-net` remains at `0.3.1`.
|
||||
|
||||
- Tag-driven CI verifies release packages once, then publishes to crates.io
|
||||
and the Furumi Gitea Cargo mirror in parallel jobs without recompiling.
|
||||
Already published versions are skipped independently in each registry.
|
||||
Authentication uses `CARGO_REGISTRY_TOKEN` and `CARGO_REGISTRIES_GITEA_TOKEN`.
|
||||
|
||||
### Podcast discovery (`podcast_dht`)
|
||||
|
||||
- `music-dht 0.6.0`: new `music_dht::podcast_dht` overlay for public podcast
|
||||
discovery on its own ALPN `furumi-fd/podcast-dht/1` (version 1) and its own
|
||||
@@ -38,7 +48,7 @@
|
||||
- The music DHT (`music-dht-v5`), `federation-net` and device-sync protocol v4
|
||||
are unchanged. See "Public discovery (podcast_dht)" in `PODCASTS.md`.
|
||||
|
||||
## Unreleased — podcast contract
|
||||
### Podcast contract and storage
|
||||
|
||||
- `music-dht 0.6.0` (breaking): new `music_dht::podcast` module with the shared
|
||||
podcast contract: canonical `FeedUrl`, `PodcastId`/`EpisodeId` derivation
|
||||
|
||||
@@ -50,13 +50,13 @@ crates/
|
||||
|
||||
## Using the libraries
|
||||
|
||||
Until crates are published to a registry, depend on the repository directly:
|
||||
Use the published crates from crates.io:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
federation-net = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
music-dht = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
furumi-library = { git = "https://gt.hexor.cy/ab/frid.git" }
|
||||
federation-net = "0.3.1"
|
||||
music-dht = "0.6.0"
|
||||
furumi-library = "0.3.0"
|
||||
```
|
||||
|
||||
Start with the crate documentation:
|
||||
@@ -80,6 +80,48 @@ The integration tests open real local sockets and establish iroh connections.
|
||||
Some rendezvous tests also use public discovery infrastructure, so a restricted
|
||||
sandbox may need explicit network permission.
|
||||
|
||||
## Publishing
|
||||
|
||||
Add these repository Actions secrets:
|
||||
|
||||
- `CARGO_REGISTRY_TOKEN`: a crates.io API token with publish access to
|
||||
`federation-net`, `music-dht` and `furumi-library`.
|
||||
- `CARGO_REGISTRIES_GITEA_TOKEN`: a raw Gitea personal access token with
|
||||
`write:package` and permission to publish under the `Furumi` organization.
|
||||
Create it under user Settings → Applications. The workflow adds the
|
||||
required `Bearer` prefix itself; no username secret is needed.
|
||||
|
||||
Update changed crate versions, workspace dependency requirements, `Cargo.lock`
|
||||
and `CHANGELOG.md`, then merge to `main`. Create `<crate>-v<version>` tags for
|
||||
the changed crates and one workspace release tag, for example:
|
||||
|
||||
```bash
|
||||
git tag -a music-dht-v0.6.0 -m 'music-dht 0.6.0'
|
||||
git tag -a furumi-library-v0.3.0 -m 'furumi-library 0.3.0'
|
||||
git tag -a v0.6.0 -m 'Frid podcast release'
|
||||
git push origin music-dht-v0.6.0 furumi-library-v0.3.0 v0.6.0
|
||||
```
|
||||
|
||||
Only the `v*` workspace tag triggers `.github/workflows/publish.yml`; the
|
||||
crate tags identify each package release without starting duplicate uploads.
|
||||
The workflow runs the workspace checks and verifies the release packages in
|
||||
one build job. Two publishing jobs then upload without rebuilding
|
||||
(`cargo publish --no-verify`), selecting missing versions independently in
|
||||
crates.io and the Gitea mirror at
|
||||
`https://gt.hexor.cy/api/packages/Furumi/cargo/`. The mirror includes all three
|
||||
crates, including an unchanged version missing there. Already published
|
||||
versions are skipped on retries; a registry lookup error fails the release
|
||||
instead of assuming absence. Mirrored manifests retain crates.io dependency
|
||||
references, so the mirror waits up to ten minutes for new workspace
|
||||
dependencies to appear in crates.io before packaging its uploads.
|
||||
|
||||
Preview the selection with `python3 scripts/publish_crates.py --plan`, or
|
||||
verify the packages without uploading with `--dry-run`. Add `--registry gitea`
|
||||
to preview or publish to the mirror. If a secret was
|
||||
missing or a publish failed, fix the cause and rerun the existing tag workflow;
|
||||
do not move release tags. Manual dispatch must select the release tag as its
|
||||
ref. Publication is performed by CI, not by the tagging commands above.
|
||||
|
||||
## License
|
||||
|
||||
Frid is released under the
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Publish missing workspace versions together; Cargo orders their dependencies."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
INDEXES = {
|
||||
"crates-io": "https://index.crates.io/",
|
||||
"gitea": "https://gt.hexor.cy/api/packages/Furumi/cargo/",
|
||||
}
|
||||
|
||||
|
||||
def index_path(name):
|
||||
name = name.lower()
|
||||
if len(name) < 3:
|
||||
return f"{len(name)}/{name}"
|
||||
if len(name) == 3:
|
||||
return f"3/{name[0]}/{name}"
|
||||
return f"{name[:2]}/{name[2:4]}/{name}"
|
||||
|
||||
|
||||
def is_published(name, version, registry="crates-io"):
|
||||
request = urllib.request.Request(
|
||||
INDEXES[registry] + index_path(name),
|
||||
headers={"User-Agent": "frid-release-ci", "Cache-Control": "no-cache"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
versions = [json.loads(line) for line in response.read().splitlines()]
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 404:
|
||||
return False
|
||||
raise
|
||||
return any(row["vers"] == version for row in versions)
|
||||
|
||||
|
||||
def unpublished_packages(metadata, registry="crates-io"):
|
||||
members = set(metadata["workspace_members"])
|
||||
packages = []
|
||||
for package in sorted(metadata["packages"], key=lambda item: item["name"]):
|
||||
registries = package["publish"]
|
||||
if package["id"] not in members or (
|
||||
registries is not None and registry not in registries
|
||||
):
|
||||
continue
|
||||
name, version = package["name"], package["version"]
|
||||
if is_published(name, version, registry):
|
||||
print(f"Skip {name} {version}: already published", flush=True)
|
||||
else:
|
||||
print(f"Publish {name} {version}", flush=True)
|
||||
packages.append(name)
|
||||
return packages
|
||||
|
||||
|
||||
def wait_for_mirror_dependencies(metadata, selected):
|
||||
# Mirrored manifests retain their crates.io dependencies. In the parallel
|
||||
# upload job, wait for new workspace dependencies to reach crates.io before
|
||||
# Cargo resolves the mirror's packages (it does not compile them again).
|
||||
members = {p["name"]: p for p in metadata["packages"] if p["id"] in metadata["workspace_members"]}
|
||||
required = {
|
||||
(dep["name"], members[dep["name"]]["version"])
|
||||
for name in selected
|
||||
for dep in members[name]["dependencies"]
|
||||
if dep.get("path") and dep["name"] in members and dep["kind"] != "dev"
|
||||
}
|
||||
deadline = time.monotonic() + 600
|
||||
while required:
|
||||
required = {(name, version) for name, version in required if not is_published(name, version)}
|
||||
if not required:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError(f"Workspace dependencies did not reach crates.io: {sorted(required)}")
|
||||
print(f"Waiting for crates.io dependencies before mirroring: {sorted(required)}", flush=True)
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--plan", action="store_true", help="list packages without publishing")
|
||||
parser.add_argument("--dry-run", action="store_true", help="verify packages without uploading")
|
||||
parser.add_argument("--registry", choices=INDEXES, default="crates-io")
|
||||
parser.add_argument("--no-verify", action="store_true", help="publish packages already verified by CI without rebuilding")
|
||||
args = parser.parse_args()
|
||||
token_variable = "CARGO_REGISTRY_TOKEN" if args.registry == "crates-io" else "CARGO_REGISTRIES_GITEA_TOKEN"
|
||||
if not (args.plan or args.dry_run) and not os.environ.get(token_variable):
|
||||
parser.error(f"{token_variable} must be set for publishing")
|
||||
metadata = json.loads(subprocess.check_output(
|
||||
["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"],
|
||||
text=True,
|
||||
))
|
||||
packages = unpublished_packages(metadata, args.registry)
|
||||
if args.plan or not packages:
|
||||
return
|
||||
if args.registry == "gitea":
|
||||
wait_for_mirror_dependencies(metadata, packages)
|
||||
command = ["cargo", "publish", "--registry", args.registry, "--locked", "--all-features"]
|
||||
for name in packages:
|
||||
command.extend(["--package", name])
|
||||
if args.dry_run:
|
||||
command.append("--dry-run")
|
||||
if args.no_verify:
|
||||
command.append("--no-verify")
|
||||
# Multi-package publishing verifies against a temporary local registry,
|
||||
# then uploads and waits for dependencies to appear before their users.
|
||||
subprocess.run(command, check=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,82 @@
|
||||
import io
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from contextlib import redirect_stdout
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import publish_crates
|
||||
|
||||
|
||||
class PublishCratesTests(unittest.TestCase):
|
||||
def test_publish_job_does_not_rebuild_verified_packages(self):
|
||||
with patch("sys.argv", ["publish_crates.py", "--no-verify"]):
|
||||
with patch.dict(publish_crates.os.environ, {"CARGO_REGISTRY_TOKEN": "test-only"}):
|
||||
with patch.object(publish_crates.subprocess, "check_output", return_value=json.dumps({})):
|
||||
with patch.object(publish_crates, "unpublished_packages", return_value=["music-dht", "furumi-library"]):
|
||||
with patch.object(publish_crates.subprocess, "run") as run:
|
||||
publish_crates.main()
|
||||
run.assert_called_once_with([
|
||||
"cargo", "publish", "--registry", "crates-io", "--locked", "--all-features",
|
||||
"--package", "music-dht", "--package", "furumi-library", "--no-verify",
|
||||
], check=True)
|
||||
|
||||
def test_existing_and_private_versions_are_excluded_on_retry(self):
|
||||
packages = [
|
||||
{"id": "net", "name": "federation-net", "version": "0.3.1", "publish": None},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "publish": None},
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "publish": None},
|
||||
{"id": "demo", "name": "demo", "version": "0.1.0", "publish": []},
|
||||
{"id": "internal", "name": "internal", "version": "0.1.0", "publish": ["private"]},
|
||||
{"id": "external", "name": "external", "version": "0.1.0", "publish": None},
|
||||
]
|
||||
metadata = {"packages": packages, "workspace_members": [p["id"] for p in packages[:-1]]}
|
||||
for published, expected in [
|
||||
({"federation-net"}, ["furumi-library", "music-dht"]),
|
||||
({"federation-net", "music-dht"}, ["furumi-library"]),
|
||||
]:
|
||||
with self.subTest(published=published), redirect_stdout(io.StringIO()):
|
||||
with patch.object(publish_crates, "is_published", side_effect=lambda name, *args: name in published):
|
||||
self.assertEqual(publish_crates.unpublished_packages(metadata), expected)
|
||||
|
||||
def test_mirror_waits_for_workspace_dependencies_without_building(self):
|
||||
metadata = {"workspace_members": ["lib", "dht"], "packages": [
|
||||
{"id": "lib", "name": "furumi-library", "version": "0.3.0", "dependencies": [
|
||||
{"name": "music-dht", "path": "/workspace/music-dht", "kind": None},
|
||||
]},
|
||||
{"id": "dht", "name": "music-dht", "version": "0.6.0", "dependencies": []},
|
||||
]}
|
||||
with patch.object(publish_crates, "is_published", side_effect=[False, True]) as published:
|
||||
with patch.object(publish_crates.time, "sleep") as sleep, redirect_stdout(io.StringIO()):
|
||||
publish_crates.wait_for_mirror_dependencies(metadata, ["furumi-library"])
|
||||
self.assertEqual(published.call_count, 2)
|
||||
sleep.assert_called_once_with(10)
|
||||
|
||||
def test_gitea_uses_its_own_index(self):
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.3.1"}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response) as fetch:
|
||||
self.assertTrue(publish_crates.is_published("federation-net", "0.3.1", "gitea"))
|
||||
self.assertEqual(fetch.call_args.args[0].full_url,
|
||||
"https://gt.hexor.cy/api/packages/Furumi/cargo/fe/de/federation-net")
|
||||
|
||||
def test_registry_errors_do_not_look_like_unpublished_versions(self):
|
||||
for status in (403, 429, 500):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", status, "error", {}, None)
|
||||
with self.subTest(status=status), patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
with self.assertRaises(urllib.error.HTTPError):
|
||||
publish_crates.is_published("music-dht", "0.6.0")
|
||||
|
||||
def test_missing_crate_and_existing_version(self):
|
||||
error = urllib.error.HTTPError("https://index.crates.io/test", 404, "missing", {}, None)
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", side_effect=error):
|
||||
self.assertFalse(publish_crates.is_published("new-crate", "0.1.0"))
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"vers":"0.6.0","yanked":false}\n'
|
||||
with patch.object(publish_crates.urllib.request, "urlopen", return_value=response):
|
||||
self.assertTrue(publish_crates.is_published("music-dht", "0.6.0"))
|
||||
self.assertFalse(publish_crates.is_published("music-dht", "0.7.0"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user