Phase 2d (0.6.1-beta): route manifest + blob ops through file_holders
Switch ALL propagation-decision reads to the flat holder set. push_manifest_to_downstream now targets file_holders instead of blob_downstream. ManifestPush receive-side relay likewise — known holders fan out to up to 5 most-recent peers instead of a directional tree. Blob delete notices: single flat fan-out to file_holders; the legacy upstream_node tree-healing field is emitted as None (wire-stable via serde default) and ignored on receive — the post-0.6 flat model doesn't need sender-role distinction. send_blob_delete_notices keeps its Option<&Upstream> parameter as an unused placeholder for signature stability with the call sites in this commit. Other reads migrated: - blob fetch cascade: step 2 now tries "known holders" (up to 5) instead of a single upstream - manifest refresh: downstream_count reported from file_holder_count - web/http post holder enumeration - Worm search post/blob holder fallback (both connection.rs paths) - DeleteRecord fan-out rewires to file_holders - Under-replication replication check: < 2 holders Storage additions: - get_file_holder_count(file_id) - remove_file_holder(file_id, peer_id) Legacy upstream/downstream writes are still happening from Phase 2b; those + the tables themselves go in 2e. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
3a0d2e93ab
commit
60463d1817
6 changed files with 103 additions and 128 deletions
|
|
@ -2828,13 +2828,9 @@ impl ConnectionManager {
|
||||||
if store.get_post_with_visibility(post_id).ok().flatten().is_some() {
|
if store.get_post_with_visibility(post_id).ok().flatten().is_some() {
|
||||||
Some(self.our_node_id)
|
Some(self.our_node_id)
|
||||||
} else {
|
} else {
|
||||||
// CDN tree: do any of our downstream hosts have it?
|
// Any known holder of this post?
|
||||||
let downstream = store.get_post_downstream(post_id).unwrap_or_default();
|
let holders = store.get_file_holders(post_id).unwrap_or_default();
|
||||||
if !downstream.is_empty() {
|
holders.first().map(|(nid, _)| *nid)
|
||||||
Some(downstream[0])
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
post_holder = found;
|
post_holder = found;
|
||||||
|
|
@ -2848,9 +2844,9 @@ impl ConnectionManager {
|
||||||
// Check CDN: do we know who has it via blob post ownership?
|
// Check CDN: do we know who has it via blob post ownership?
|
||||||
let store = self.storage.get().await;
|
let store = self.storage.get().await;
|
||||||
if let Ok(Some(pid)) = store.get_blob_post_id(blob_id) {
|
if let Ok(Some(pid)) = store.get_blob_post_id(blob_id) {
|
||||||
let downstream = store.get_post_downstream(&pid).unwrap_or_default();
|
let holders = store.get_file_holders(&pid).unwrap_or_default();
|
||||||
if !downstream.is_empty() {
|
if let Some((nid, _)) = holders.first() {
|
||||||
blob_holder = Some(downstream[0]);
|
blob_holder = Some(*nid);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -4889,7 +4885,7 @@ impl ConnectionManager {
|
||||||
let cm = conn_mgr.lock().await;
|
let cm = conn_mgr.lock().await;
|
||||||
|
|
||||||
// Collect blob CIDs + CDN peers before async work
|
// Collect blob CIDs + CDN peers before async work
|
||||||
let mut blob_cleanup: Vec<([u8; 32], Vec<(NodeId, Vec<String>)>, Option<(NodeId, Vec<String>)>)> = Vec::new();
|
let mut blob_cleanup: Vec<([u8; 32], Vec<(NodeId, Vec<String>)>)> = Vec::new();
|
||||||
{
|
{
|
||||||
let storage = cm.storage.get().await;
|
let storage = cm.storage.get().await;
|
||||||
for dr in &payload.records {
|
for dr in &payload.records {
|
||||||
|
|
@ -4897,9 +4893,8 @@ impl ConnectionManager {
|
||||||
// Collect blobs for CDN cleanup before deleting
|
// Collect blobs for CDN cleanup before deleting
|
||||||
let blob_cids = storage.get_blobs_for_post(&dr.post_id).unwrap_or_default();
|
let blob_cids = storage.get_blobs_for_post(&dr.post_id).unwrap_or_default();
|
||||||
for cid in blob_cids {
|
for cid in blob_cids {
|
||||||
let downstream = storage.get_blob_downstream(&cid).unwrap_or_default();
|
let holders = storage.get_file_holders(&cid).unwrap_or_default();
|
||||||
let upstream = storage.get_blob_upstream(&cid).ok().flatten();
|
blob_cleanup.push((cid, holders));
|
||||||
blob_cleanup.push((cid, downstream, upstream));
|
|
||||||
}
|
}
|
||||||
let _ = storage.store_delete(dr);
|
let _ = storage.store_delete(dr);
|
||||||
let _ = storage.apply_delete(dr);
|
let _ = storage.apply_delete(dr);
|
||||||
|
|
@ -4915,18 +4910,11 @@ impl ConnectionManager {
|
||||||
|
|
||||||
// Gather connections for CDN delete notices under lock, then send outside
|
// Gather connections for CDN delete notices under lock, then send outside
|
||||||
let mut delete_notices: Vec<(iroh::endpoint::Connection, crate::protocol::BlobDeleteNoticePayload)> = Vec::new();
|
let mut delete_notices: Vec<(iroh::endpoint::Connection, crate::protocol::BlobDeleteNoticePayload)> = Vec::new();
|
||||||
for (cid, downstream, upstream) in &blob_cleanup {
|
for (cid, holders) in &blob_cleanup {
|
||||||
let upstream_info = upstream.as_ref().map(|(nid, addrs)| PeerWithAddress { n: hex::encode(nid), a: addrs.clone() });
|
let payload = crate::protocol::BlobDeleteNoticePayload { cid: *cid, upstream_node: None };
|
||||||
let ds_payload = crate::protocol::BlobDeleteNoticePayload { cid: *cid, upstream_node: upstream_info };
|
for (peer, _addrs) in holders {
|
||||||
for (ds_nid, _) in downstream {
|
if let Some(pc) = cm.connections_ref().get(peer) {
|
||||||
if let Some(pc) = cm.connections_ref().get(ds_nid) {
|
delete_notices.push((pc.connection.clone(), payload.clone()));
|
||||||
delete_notices.push((pc.connection.clone(), ds_payload.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some((up_nid, _)) = upstream {
|
|
||||||
let up_payload = crate::protocol::BlobDeleteNoticePayload { cid: *cid, upstream_node: None };
|
|
||||||
if let Some(pc) = cm.connections_ref().get(up_nid) {
|
|
||||||
delete_notices.push((pc.connection.clone(), up_payload));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -5106,15 +5094,15 @@ impl ConnectionManager {
|
||||||
);
|
);
|
||||||
stored_entries.push(entry.clone());
|
stored_entries.push(entry.clone());
|
||||||
}
|
}
|
||||||
// Gather downstream peers for relay before dropping locks
|
// Gather file holders for relay before dropping locks
|
||||||
let mut relay_targets: Vec<(NodeId, crate::protocol::ManifestPushPayload)> = Vec::new();
|
let mut relay_targets: Vec<(NodeId, crate::protocol::ManifestPushPayload)> = Vec::new();
|
||||||
for entry in &stored_entries {
|
for entry in &stored_entries {
|
||||||
let downstream = storage.get_blob_downstream(&entry.cid).unwrap_or_default();
|
let holders = storage.get_file_holders(&entry.cid).unwrap_or_default();
|
||||||
for (ds_nid, _) in downstream {
|
for (peer, _addrs) in holders {
|
||||||
if ds_nid == remote_node_id {
|
if peer == remote_node_id {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
relay_targets.push((ds_nid, crate::protocol::ManifestPushPayload {
|
relay_targets.push((peer, crate::protocol::ManifestPushPayload {
|
||||||
manifests: vec![entry.clone()],
|
manifests: vec![entry.clone()],
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
@ -5315,32 +5303,14 @@ impl ConnectionManager {
|
||||||
let storage = cm.storage.get().await;
|
let storage = cm.storage.get().await;
|
||||||
let cid = payload.cid;
|
let cid = payload.cid;
|
||||||
|
|
||||||
// Check if sender was our upstream for this blob
|
// Flat-holder model: drop the sender as a holder of this file.
|
||||||
let was_upstream = storage.get_blob_upstream(&cid).ok().flatten()
|
// The author's DeleteRecord (separate signed message) is what
|
||||||
.map(|(nid, _)| nid == remote_node_id)
|
// triggers the actual blob removal for followers.
|
||||||
.unwrap_or(false);
|
let _ = storage.remove_file_holder(&cid, &remote_node_id);
|
||||||
|
|
||||||
if was_upstream {
|
|
||||||
// Sender was our upstream — clear it
|
|
||||||
let _ = storage.remove_blob_upstream(&cid);
|
|
||||||
|
|
||||||
// If they provided their upstream, store it as our new upstream
|
|
||||||
if let Some(ref new_up) = payload.upstream_node {
|
|
||||||
if let Ok(nid_bytes) = hex::decode(&new_up.n) {
|
|
||||||
if let Ok(nid) = <[u8; 32]>::try_from(nid_bytes.as_slice()) {
|
|
||||||
let _ = storage.store_blob_upstream(&cid, &nid, &new_up.a);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Sender was our downstream — remove them
|
|
||||||
let _ = storage.remove_blob_downstream(&cid, &remote_node_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
peer = hex::encode(remote_node_id),
|
peer = hex::encode(remote_node_id),
|
||||||
cid = hex::encode(cid),
|
cid = hex::encode(cid),
|
||||||
was_upstream,
|
|
||||||
"Received blob delete notice"
|
"Received blob delete notice"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -5745,21 +5715,28 @@ impl ConnectionManager {
|
||||||
let storage = storage.get().await;
|
let storage = storage.get().await;
|
||||||
let manifest: Option<crate::types::CdnManifest> = storage.get_cdn_manifest(&payload.cid).ok().flatten().and_then(|json| {
|
let manifest: Option<crate::types::CdnManifest> = storage.get_cdn_manifest(&payload.cid).ok().flatten().and_then(|json| {
|
||||||
if let Ok(am) = serde_json::from_str::<crate::types::AuthorManifest>(&json) {
|
if let Ok(am) = serde_json::from_str::<crate::types::AuthorManifest>(&json) {
|
||||||
let ds_count = storage.get_blob_downstream_count(&payload.cid).unwrap_or(0);
|
let ds_count = storage.get_file_holder_count(&payload.cid).unwrap_or(0);
|
||||||
Some(crate::types::CdnManifest { author_manifest: am, host: our_node_id, host_addresses: vec![], source: our_node_id, source_addresses: vec![], downstream_count: ds_count })
|
Some(crate::types::CdnManifest { author_manifest: am, host: our_node_id, host_addresses: vec![], source: our_node_id, source_addresses: vec![], downstream_count: ds_count })
|
||||||
} else { serde_json::from_str(&json).ok() }
|
} else { serde_json::from_str(&json).ok() }
|
||||||
});
|
});
|
||||||
let (cdn_registered, cdn_redirect_peers) = if !payload.requester_addresses.is_empty() {
|
let (cdn_registered, cdn_redirect_peers) = if !payload.requester_addresses.is_empty() {
|
||||||
let ok = storage.add_blob_downstream(&payload.cid, &remote_node_id, &payload.requester_addresses).unwrap_or(false);
|
let prior_count = storage.get_file_holder_count(&payload.cid).unwrap_or(0);
|
||||||
let _ = storage.touch_file_holder(
|
let _ = storage.touch_file_holder(
|
||||||
&payload.cid,
|
&payload.cid,
|
||||||
&remote_node_id,
|
&remote_node_id,
|
||||||
&payload.requester_addresses,
|
&payload.requester_addresses,
|
||||||
crate::storage::HolderDirection::Sent,
|
crate::storage::HolderDirection::Sent,
|
||||||
);
|
);
|
||||||
if ok { (true, vec![]) } else {
|
// If we already had 5 holders before adding this one, the
|
||||||
let downstream = storage.get_blob_downstream(&payload.cid).unwrap_or_default();
|
// requester should consult them too for CDN lookups.
|
||||||
let redirects: Vec<PeerWithAddress> = downstream.into_iter().map(|(nid, addrs)| PeerWithAddress { n: hex::encode(nid), a: addrs }).collect();
|
if prior_count < 5 {
|
||||||
|
(true, vec![])
|
||||||
|
} else {
|
||||||
|
let holders = storage.get_file_holders(&payload.cid).unwrap_or_default();
|
||||||
|
let redirects: Vec<PeerWithAddress> = holders.into_iter()
|
||||||
|
.filter(|(nid, _)| *nid != remote_node_id)
|
||||||
|
.map(|(nid, addrs)| PeerWithAddress { n: hex::encode(nid), a: addrs })
|
||||||
|
.collect();
|
||||||
(false, redirects)
|
(false, redirects)
|
||||||
}
|
}
|
||||||
} else { (false, vec![]) };
|
} else { (false, vec![]) };
|
||||||
|
|
@ -5786,7 +5763,7 @@ impl ConnectionManager {
|
||||||
Some(json) => {
|
Some(json) => {
|
||||||
let manifest = if let Ok(am) = serde_json::from_str::<crate::types::AuthorManifest>(&json) {
|
let manifest = if let Ok(am) = serde_json::from_str::<crate::types::AuthorManifest>(&json) {
|
||||||
if am.updated_at > payload.current_updated_at {
|
if am.updated_at > payload.current_updated_at {
|
||||||
let ds_count = store.get_blob_downstream_count(&payload.cid).unwrap_or(0);
|
let ds_count = store.get_file_holder_count(&payload.cid).unwrap_or(0);
|
||||||
Some(crate::types::CdnManifest { author_manifest: am, host: our_node_id, host_addresses: vec![], source: our_node_id, source_addresses: vec![], downstream_count: ds_count })
|
Some(crate::types::CdnManifest { author_manifest: am, host: our_node_id, host_addresses: vec![], source: our_node_id, source_addresses: vec![], downstream_count: ds_count })
|
||||||
} else { None }
|
} else { None }
|
||||||
} else { None };
|
} else { None };
|
||||||
|
|
@ -7758,8 +7735,8 @@ impl ConnectionActor {
|
||||||
if s.get_post_with_visibility(post_id).ok().flatten().is_some() {
|
if s.get_post_with_visibility(post_id).ok().flatten().is_some() {
|
||||||
post_holder = Some(ctx.our_node_id);
|
post_holder = Some(ctx.our_node_id);
|
||||||
} else {
|
} else {
|
||||||
let downstream = s.get_post_downstream(post_id).unwrap_or_default();
|
let holders = s.get_file_holders(post_id).unwrap_or_default();
|
||||||
if !downstream.is_empty() { post_holder = Some(downstream[0]); }
|
if let Some((nid, _)) = holders.first() { post_holder = Some(*nid); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -7769,8 +7746,8 @@ impl ConnectionActor {
|
||||||
} else {
|
} else {
|
||||||
let s = ctx.storage.get().await;
|
let s = ctx.storage.get().await;
|
||||||
if let Ok(Some(pid)) = s.get_blob_post_id(blob_id) {
|
if let Ok(Some(pid)) = s.get_blob_post_id(blob_id) {
|
||||||
let downstream = s.get_post_downstream(&pid).unwrap_or_default();
|
let holders = s.get_file_holders(&pid).unwrap_or_default();
|
||||||
if !downstream.is_empty() { blob_holder = Some(downstream[0]); }
|
if let Some((nid, _)) = holders.first() { blob_holder = Some(*nid); }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -378,7 +378,11 @@ async fn try_redirect(
|
||||||
Ok(Some((_, PostVisibility::Public))) => {}
|
Ok(Some((_, PostVisibility::Public))) => {}
|
||||||
_ => return false, // not found or not public — hard close
|
_ => return false, // not found or not public — hard close
|
||||||
}
|
}
|
||||||
store.get_post_downstream(post_id).unwrap_or_default()
|
store.get_file_holders(post_id)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.map(|(nid, _addrs)| nid)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
};
|
};
|
||||||
|
|
||||||
// Get addresses for downstream peers
|
// Get addresses for downstream peers
|
||||||
|
|
|
||||||
|
|
@ -1015,15 +1015,16 @@ impl Network {
|
||||||
sent
|
sent
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Push updated manifests to all downstream peers for a given CID.
|
/// Push an updated manifest to all known holders of the file (flat set,
|
||||||
|
/// up to 5 most-recent). Replaces the legacy downstream-tree push.
|
||||||
pub async fn push_manifest_to_downstream(
|
pub async fn push_manifest_to_downstream(
|
||||||
&self,
|
&self,
|
||||||
cid: &[u8; 32],
|
cid: &[u8; 32],
|
||||||
manifest: &crate::types::CdnManifest,
|
manifest: &crate::types::CdnManifest,
|
||||||
) -> usize {
|
) -> usize {
|
||||||
let downstream = {
|
let holders = {
|
||||||
let storage = self.storage.get().await;
|
let storage = self.storage.get().await;
|
||||||
storage.get_blob_downstream(cid).unwrap_or_default()
|
storage.get_file_holders(cid).unwrap_or_default()
|
||||||
};
|
};
|
||||||
let payload = crate::protocol::ManifestPushPayload {
|
let payload = crate::protocol::ManifestPushPayload {
|
||||||
manifests: vec![crate::protocol::ManifestPushEntry {
|
manifests: vec![crate::protocol::ManifestPushEntry {
|
||||||
|
|
@ -1032,15 +1033,14 @@ impl Network {
|
||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
let mut sent = 0;
|
let mut sent = 0;
|
||||||
for (ds_nid, ds_addrs) in &downstream {
|
for (peer, peer_addrs) in &holders {
|
||||||
if self.send_to_peer_uni(ds_nid, MessageType::ManifestPush, &payload).await.is_ok() {
|
if self.send_to_peer_uni(peer, MessageType::ManifestPush, &payload).await.is_ok() {
|
||||||
sent += 1;
|
sent += 1;
|
||||||
// We pushed this file's manifest → downstream peer now holds it.
|
|
||||||
let storage = self.storage.get().await;
|
let storage = self.storage.get().await;
|
||||||
let _ = storage.touch_file_holder(
|
let _ = storage.touch_file_holder(
|
||||||
cid,
|
cid,
|
||||||
ds_nid,
|
peer,
|
||||||
ds_addrs,
|
peer_addrs,
|
||||||
crate::storage::HolderDirection::Sent,
|
crate::storage::HolderDirection::Sent,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -1048,46 +1048,25 @@ impl Network {
|
||||||
sent
|
sent
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send blob delete notices to downstream and upstream peers.
|
/// Send blob delete notices to all known holders of a file.
|
||||||
/// Downstream peers receive our upstream info for tree healing.
|
/// Second argument kept as Option for signature stability; flat-holder
|
||||||
/// Upstream peers receive no upstream info (just "remove me as downstream").
|
/// model doesn't need separate upstream handling.
|
||||||
pub async fn send_blob_delete_notices(
|
pub async fn send_blob_delete_notices(
|
||||||
&self,
|
&self,
|
||||||
cid: &[u8; 32],
|
cid: &[u8; 32],
|
||||||
downstream: &[(NodeId, Vec<String>)],
|
holders: &[(NodeId, Vec<String>)],
|
||||||
upstream: Option<&(NodeId, Vec<String>)>,
|
_legacy_upstream: Option<&(NodeId, Vec<String>)>,
|
||||||
) -> usize {
|
) -> usize {
|
||||||
let upstream_info = upstream.map(|(nid, addrs)| {
|
let payload = crate::protocol::BlobDeleteNoticePayload {
|
||||||
crate::types::PeerWithAddress {
|
|
||||||
n: hex::encode(nid),
|
|
||||||
a: addrs.clone(),
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let mut sent = 0;
|
|
||||||
|
|
||||||
// Notify downstream (with upstream info for tree healing)
|
|
||||||
let ds_payload = crate::protocol::BlobDeleteNoticePayload {
|
|
||||||
cid: *cid,
|
|
||||||
upstream_node: upstream_info,
|
|
||||||
};
|
|
||||||
for (ds_nid, _) in downstream {
|
|
||||||
if self.send_to_peer_uni(ds_nid, MessageType::BlobDeleteNotice, &ds_payload).await.is_ok() {
|
|
||||||
sent += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Notify upstream (no upstream info)
|
|
||||||
if let Some((up_nid, _)) = upstream {
|
|
||||||
let up_payload = crate::protocol::BlobDeleteNoticePayload {
|
|
||||||
cid: *cid,
|
cid: *cid,
|
||||||
upstream_node: None,
|
upstream_node: None,
|
||||||
};
|
};
|
||||||
if self.send_to_peer_uni(up_nid, MessageType::BlobDeleteNotice, &up_payload).await.is_ok() {
|
let mut sent = 0;
|
||||||
|
for (peer, _addrs) in holders {
|
||||||
|
if self.send_to_peer_uni(peer, MessageType::BlobDeleteNotice, &payload).await.is_ok() {
|
||||||
sent += 1;
|
sent += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sent
|
sent
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1384,16 +1384,17 @@ impl Node {
|
||||||
// Collect redirect peers from responses in case we need them later
|
// Collect redirect peers from responses in case we need them later
|
||||||
let mut redirect_peers: Vec<crate::types::PeerWithAddress> = Vec::new();
|
let mut redirect_peers: Vec<crate::types::PeerWithAddress> = Vec::new();
|
||||||
|
|
||||||
// 2. Try existing upstream (if we previously fetched this blob)
|
// 2. Try known holders (up to 5 most-recent peers we've interacted
|
||||||
let upstream = {
|
// with about this file).
|
||||||
|
let known_holders = {
|
||||||
let storage = self.storage.get().await;
|
let storage = self.storage.get().await;
|
||||||
storage.get_blob_upstream(cid)?
|
storage.get_file_holders(cid).unwrap_or_default()
|
||||||
};
|
};
|
||||||
if let Some((upstream_nid, _upstream_addrs)) = upstream {
|
for (holder_nid, _addrs) in &known_holders {
|
||||||
match self.fetch_blob_from_peer(cid, &upstream_nid, post_id, author, mime_type, created_at).await {
|
match self.fetch_blob_from_peer(cid, holder_nid, post_id, author, mime_type, created_at).await {
|
||||||
Ok(Some(data)) => return Ok(Some(data)),
|
Ok(Some(data)) => return Ok(Some(data)),
|
||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Err(e) => warn!(error = %e, "blob fetch from upstream failed"),
|
Err(e) => warn!(error = %e, "blob fetch from known holder failed"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1992,14 +1993,13 @@ impl Node {
|
||||||
signature,
|
signature,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Collect blob CIDs + CDN peers before cleanup
|
// Collect blob CIDs + known holders before cleanup (for delete notices)
|
||||||
let blob_cdn_info: Vec<([u8; 32], Vec<(NodeId, Vec<String>)>, Option<(NodeId, Vec<String>)>)> = {
|
let blob_cdn_info: Vec<([u8; 32], Vec<(NodeId, Vec<String>)>, Option<(NodeId, Vec<String>)>)> = {
|
||||||
let storage = self.storage.get().await;
|
let storage = self.storage.get().await;
|
||||||
let cids = storage.get_blobs_for_post(post_id).unwrap_or_default();
|
let cids = storage.get_blobs_for_post(post_id).unwrap_or_default();
|
||||||
cids.into_iter().map(|cid| {
|
cids.into_iter().map(|cid| {
|
||||||
let downstream = storage.get_blob_downstream(&cid).unwrap_or_default();
|
let holders = storage.get_file_holders(&cid).unwrap_or_default();
|
||||||
let upstream = storage.get_blob_upstream(&cid).ok().flatten();
|
(cid, holders, None::<(NodeId, Vec<String>)>)
|
||||||
(cid, downstream, upstream)
|
|
||||||
}).collect()
|
}).collect()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -3119,10 +3119,10 @@ impl Node {
|
||||||
&cdn_manifest.author_manifest.author,
|
&cdn_manifest.author_manifest.author,
|
||||||
cdn_manifest.author_manifest.updated_at,
|
cdn_manifest.author_manifest.updated_at,
|
||||||
);
|
);
|
||||||
// Relay to our downstream
|
// Relay to known holders (flat set)
|
||||||
let downstream = s.get_blob_downstream(cid).unwrap_or_default();
|
let holders = s.get_file_holders(cid).unwrap_or_default();
|
||||||
drop(s);
|
drop(s);
|
||||||
if !downstream.is_empty() {
|
if !holders.is_empty() {
|
||||||
network.push_manifest_to_downstream(cid, &cdn_manifest).await;
|
network.push_manifest_to_downstream(cid, &cdn_manifest).await;
|
||||||
}
|
}
|
||||||
tracing::debug!(
|
tracing::debug!(
|
||||||
|
|
@ -3286,18 +3286,16 @@ impl Node {
|
||||||
compute_blob_priority_standalone(candidate, &self.node_id, follows, audience_members, now_ms)
|
compute_blob_priority_standalone(candidate, &self.node_id, follows, audience_members, now_ms)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete a blob with CDN notifications to upstream/downstream.
|
/// Delete a blob with CDN notifications to known holders.
|
||||||
pub async fn delete_blob_with_cdn_notify(&self, cid: &[u8; 32]) -> anyhow::Result<()> {
|
pub async fn delete_blob_with_cdn_notify(&self, cid: &[u8; 32]) -> anyhow::Result<()> {
|
||||||
// Gather CDN peers before cleanup
|
// Gather known holders before cleanup
|
||||||
let (downstream, upstream) = {
|
let holders = {
|
||||||
let storage = self.storage.get().await;
|
let storage = self.storage.get().await;
|
||||||
let ds = storage.get_blob_downstream(cid).unwrap_or_default();
|
storage.get_file_holders(cid).unwrap_or_default()
|
||||||
let up = storage.get_blob_upstream(cid).ok().flatten();
|
|
||||||
(ds, up)
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Send CDN delete notices
|
// Send CDN delete notices to all holders
|
||||||
self.network.send_blob_delete_notices(cid, &downstream, upstream.as_ref()).await;
|
self.network.send_blob_delete_notices(cid, &holders, None).await;
|
||||||
|
|
||||||
// Clean up local storage
|
// Clean up local storage
|
||||||
{
|
{
|
||||||
|
|
@ -4330,10 +4328,10 @@ impl Node {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Filter to under-replicated (< 2 downstream)
|
// Filter to under-replicated (< 2 holders)
|
||||||
let mut needs_replication = Vec::new();
|
let mut needs_replication = Vec::new();
|
||||||
for pid in &recent_ids {
|
for pid in &recent_ids {
|
||||||
match storage.get_post_downstream_count(pid) {
|
match storage.get_file_holder_count(pid) {
|
||||||
Ok(count) if count < 2 => {
|
Ok(count) if count < 2 => {
|
||||||
needs_replication.push(*pid);
|
needs_replication.push(*pid);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4470,6 +4470,14 @@ impl Storage {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Count file holders (bounded at 5 by touch_file_holder's LRU cap).
|
||||||
|
pub fn get_file_holder_count(&self, file_id: &[u8; 32]) -> anyhow::Result<u32> {
|
||||||
|
let count: i64 = self.conn.prepare(
|
||||||
|
"SELECT COUNT(*) FROM file_holders WHERE file_id = ?1",
|
||||||
|
)?.query_row(params![file_id.as_slice()], |row| row.get(0))?;
|
||||||
|
Ok(count as u32)
|
||||||
|
}
|
||||||
|
|
||||||
/// Return the up-to-5 most recently interacted holders of a file.
|
/// Return the up-to-5 most recently interacted holders of a file.
|
||||||
pub fn get_file_holders(&self, file_id: &[u8; 32]) -> anyhow::Result<Vec<(NodeId, Vec<String>)>> {
|
pub fn get_file_holders(&self, file_id: &[u8; 32]) -> anyhow::Result<Vec<(NodeId, Vec<String>)>> {
|
||||||
let mut stmt = self.conn.prepare(
|
let mut stmt = self.conn.prepare(
|
||||||
|
|
@ -4504,6 +4512,15 @@ impl Storage {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Remove a single peer's holder entry for a file.
|
||||||
|
pub fn remove_file_holder(&self, file_id: &[u8; 32], peer_id: &NodeId) -> anyhow::Result<()> {
|
||||||
|
self.conn.execute(
|
||||||
|
"DELETE FROM file_holders WHERE file_id = ?1 AND peer_id = ?2",
|
||||||
|
params![file_id.as_slice(), peer_id.as_slice()],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// One-time migration: seed file_holders from the legacy upstream/downstream
|
/// One-time migration: seed file_holders from the legacy upstream/downstream
|
||||||
/// tables so a user upgrading from pre-0.6.1 doesn't start with empty holder
|
/// tables so a user upgrading from pre-0.6.1 doesn't start with empty holder
|
||||||
/// sets. Idempotent — inserts use ON CONFLICT DO NOTHING semantics via the
|
/// sets. Idempotent — inserts use ON CONFLICT DO NOTHING semantics via the
|
||||||
|
|
|
||||||
|
|
@ -132,8 +132,8 @@ async fn serve_post(stream: &mut TcpStream, path: &str, node: &Arc<Node>, browse
|
||||||
if let Some(author) = author_id {
|
if let Some(author) = author_id {
|
||||||
holders.push(author);
|
holders.push(author);
|
||||||
}
|
}
|
||||||
if let Ok(downstream) = store.get_post_downstream(&post_id) {
|
if let Ok(file_holders) = store.get_file_holders(&post_id) {
|
||||||
for peer in downstream {
|
for (peer, _addrs) in file_holders {
|
||||||
if !holders.contains(&peer) {
|
if !holders.contains(&peer) {
|
||||||
holders.push(peer);
|
holders.push(peer);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue