v0.4.0: Protocol v4 — header-driven sync, tiered engagement, multi-upstream
Protocol v4 sync overhaul: - Slim PullSyncRequest: per-author timestamps (since_ms) replace full post ID lists Request size O(follows) instead of O(posts). Backward-compatible via serde default. - Tiered pull frequency: 60s ticks, only syncs stale authors (4hr default) Full pull only on first tick (bootstrap). Most ticks skip — no stale authors. - Tiered engagement checks: frequency scales with content age 5min (<72h), 1hr (3-14d), 4hr (14-30d), 24hr (>30d) Single SQL query filters posts due for check. - Header-driven post discovery: ManifestPush triggers PostFetch for missing followed-author posts (capped 10 per manifest). CDN tree = notification system. - Multi-upstream (3 max): composite PK, priority ordering, engagement diffs sent to all upstreams, promote/remove on failure. DB schema: - follows.last_sync_ms — Self Last Encounter per author - posts.last_engagement_ms — last reaction/comment timestamp - posts.last_check_ms — last engagement check timestamp - post_upstream: single-row → 3-row with priority column Lock contention fixes: - get_blob_for_post: 3 locks → 1 - prefetch_blobs_from_peer: lock-free blob checks - fetch_engagement_from_peer: explicit lock release before I/O - serve_post: 4 locks → 2 (eliminated redundant queries) - run_replication_check: 2 locks → 1 - Badge cycle: N+2 IPC calls → 1 (get_badge_counts) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1df00eebf8
commit
bbaacf9b6c
10 changed files with 489 additions and 100 deletions
|
|
@ -1740,9 +1740,12 @@ impl Network {
|
|||
/// Pull posts from a peer (persistent if available, ephemeral otherwise).
|
||||
pub async fn pull_from_peer(&self, peer_id: &NodeId) -> anyhow::Result<PullStats> {
|
||||
let conn = self.get_connection(peer_id).await?;
|
||||
let (our_follows, our_post_ids) = {
|
||||
let (our_follows, follows_sync) = {
|
||||
let storage = self.storage.lock().await;
|
||||
(storage.list_follows()?, storage.list_post_ids()?)
|
||||
(
|
||||
storage.list_follows()?,
|
||||
storage.get_follows_with_last_sync().unwrap_or_default(),
|
||||
)
|
||||
};
|
||||
let (mut send, mut recv) = conn.open_bi().await?;
|
||||
write_typed_message(
|
||||
|
|
@ -1750,7 +1753,8 @@ impl Network {
|
|||
MessageType::PullSyncRequest,
|
||||
&PullSyncRequestPayload {
|
||||
follows: our_follows,
|
||||
have_post_ids: our_post_ids,
|
||||
have_post_ids: vec![], // v4: empty, using since_ms instead
|
||||
since_ms: follows_sync,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -1760,14 +1764,20 @@ impl Network {
|
|||
anyhow::bail!("expected PullSyncResponse, got {:?}", msg_type);
|
||||
}
|
||||
let response: PullSyncResponsePayload = read_payload(&mut recv, 64 * 1024 * 1024).await?;
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis() as u64;
|
||||
let storage = self.storage.lock().await;
|
||||
let mut posts_received = 0;
|
||||
let mut vis_updates = 0;
|
||||
for sp in response.posts {
|
||||
for sp in &response.posts {
|
||||
if !storage.is_deleted(&sp.id)? && verify_post_id(&sp.id, &sp.post) {
|
||||
if storage.store_post_with_visibility(&sp.id, &sp.post, &sp.visibility)? {
|
||||
posts_received += 1;
|
||||
}
|
||||
// Protocol v4: update last_sync_ms for the author
|
||||
let _ = storage.update_follow_last_sync(&sp.post.author, now_ms);
|
||||
}
|
||||
}
|
||||
for vu in response.visibility_updates {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue