P-Diff-Sync: merge perspective diffs into refs -> only one HC entry per commit

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/chunked_diffs.rs

@@ -1,6 +1,6 @@
 use hdk::prelude::*;
 use perspective_diff_sync_integrity::{
-    EntryTypes, LinkExpression, PerspectiveDiff,
+    EntryTypes, LinkExpression, PerspectiveDiff, PerspectiveDiffEntryReference,
 };
 
 use crate::{Hash, CHUNK_SIZE};
@@ -59,15 +59,20 @@ impl ChunkedDiffs {
             .into_iter()
             .map(|chunk_diff| {
                 debug!("ChunkedDiffs writing chunk of size: {}", chunk_diff.total_diff_number());
-                Retreiver::create_entry(EntryTypes::PerspectiveDiff(chunk_diff))
+                let diff_entry = PerspectiveDiffEntryReference::new(
+                    chunk_diff,
+                    None, // No parents for chunk entries
+                );
+                Retreiver::create_entry(EntryTypes::PerspectiveDiffEntryReference(diff_entry))
             })
             .collect() 
     }
 
     pub fn from_entries<Retreiver: PerspectiveDiffRetreiver>(hashes: Vec<Hash>) -> SocialContextResult<Self> {
         let mut diffs = Vec::new();
         for hash in hashes.into_iter() {
-            diffs.push(Retreiver::get::<PerspectiveDiff>(hash)?);
+            let diff_entry = Retreiver::get::<PerspectiveDiffEntryReference>(hash)?;
+            diffs.push(diff_entry.diff);
         }
 
         Ok(ChunkedDiffs {

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/commit.rs

@@ -42,9 +42,8 @@ pub fn commit<Retriever: PerspectiveDiffRetreiver>(
     };
 
     let now = get_now()?.time();
-    let diff_entry_create = Retriever::create_entry(EntryTypes::PerspectiveDiff(diff.clone()))?;
     let diff_entry_ref_entry = PerspectiveDiffEntryReference {
-        diff: diff_entry_create.clone(),
+        diff: diff.clone(),
         parents: initial_current_revision.clone().map(|val| vec![val.hash]),
         diffs_since_snapshot: entries_since_snapshot,
     };
@@ -57,7 +56,7 @@ pub fn commit<Retriever: PerspectiveDiffRetreiver>(
     //     diff_entry_reference
     // );
     debug!(
-        "===PerspectiveDiffSync.commit() - Profiling: Took {} to create a PerspectiveDiff",
+        "===PerspectiveDiffSync.commit() - Profiling: Took {} to create a PerspectiveDiffEntryReference",
         (after - now).num_milliseconds()
     );
 
@@ -144,12 +143,10 @@ pub fn broadcast_current<Retriever: PerspectiveDiffRetreiver>() -> SocialContext
         let current_revision = current.clone().unwrap();
         let entry_ref =
             Retriever::get::<PerspectiveDiffEntryReference>(current_revision.hash.clone())?;
-        let diff = Retriever::get::<PerspectiveDiff>(entry_ref.diff.clone())?;
 
         let signal_data = HashBroadcast {
             reference: entry_ref,
             reference_hash: current_revision.hash.clone(),
-            diff,
             broadcast_author: get_my_did()?.unwrap(),
         };
 

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/pull.rs

@@ -24,13 +24,11 @@ fn merge<Retriever: PerspectiveDiffRetreiver>(
         additions: vec![],
         removals: vec![],
     };
-    let merge_entry_hash =
-        Retriever::create_entry(EntryTypes::PerspectiveDiff(merge_diff.clone()))?;
 
     //Create the merge entry reference
     let merge_entry_reference = PerspectiveDiffEntryReference {
         parents: Some(vec![latest, current]),
-        diff: merge_entry_hash.clone(),
+        diff: merge_diff.clone(),
         diffs_since_snapshot: latest_diff.diffs_since_snapshot
             + current_diff.diffs_since_snapshot
             + 1,
@@ -159,9 +157,8 @@ pub fn pull<Retriever: PerspectiveDiffRetreiver>(
             removals: vec![],
         };
         for diff in unseen_diffs {
-            let diff_entry = Retriever::get::<PerspectiveDiff>(diff.1.diff.clone())?;
-            out.additions.append(&mut diff_entry.additions.clone());
-            out.removals.append(&mut diff_entry.removals.clone());
+            out.additions.append(&mut diff.1.diff.additions.clone());
+            out.removals.append(&mut diff.1.diff.removals.clone());
         }
         update_current_revision::<Retriever>(theirs.clone(), get_now()?)?;
         let fn_end = get_now()?.time();
@@ -178,9 +175,8 @@ pub fn pull<Retriever: PerspectiveDiffRetreiver>(
             removals: vec![],
         };
         for diff in unseen_diffs {
-            let diff_entry = Retriever::get::<PerspectiveDiff>(diff.1.diff.clone())?;
-            out.additions.append(&mut diff_entry.additions.clone());
-            out.removals.append(&mut diff_entry.removals.clone());
+            out.additions.append(&mut diff.1.diff.additions.clone());
+            out.removals.append(&mut diff.1.diff.removals.clone());
         }
 
         let merge_hash = merge::<Retriever>(theirs, current.hash)?;
@@ -230,7 +226,7 @@ pub fn handle_broadcast<Retriever: PerspectiveDiffRetreiver>(
         if diff_reference.parents == Some(vec![current_revision.hash]) {
             // debug!("===PerspectiveDiffSync.fast_forward_signal(): Revisions parent is the same as current, we can fast forward our current");
             update_current_revision::<Retriever>(revision, get_now()?)?;
-            emit_signal(broadcast.diff.clone())?;
+            emit_signal(broadcast.reference.diff.clone())?;
         };
     };
     emit_signal(broadcast)?;

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/render.rs

@@ -1,5 +1,5 @@
 use hdk::prelude::*;
-use perspective_diff_sync_integrity::PerspectiveDiff;
+use std::collections::HashSet;
 
 use crate::errors::{SocialContextError, SocialContextResult};
 use crate::link_adapter::revisions::current_revision;
@@ -22,16 +22,23 @@ pub fn render<Retriever: PerspectiveDiffRetreiver>() -> SocialContextResult<Pers
     workspace.collect_only_from_latest::<Retriever>(current.hash)?;
 
     let mut perspective = Perspective { links: vec![] };
+
+    // Collect all removals into a HashSet for O(1) lookup
+    let mut removals_set = HashSet::new();
+
     for diff_node in workspace.entry_map {
-        let diff_entry = Retriever::get::<PerspectiveDiff>(diff_node.1.diff.clone())?;
-
-        for addition in diff_entry.additions {
+        // Add all additions to the perspective
+        for addition in diff_node.1.diff.additions {
             perspective.links.push(addition);
         }
-        for removal in diff_entry.removals {
-            perspective.links.retain(|l| l != &removal);
+        // Collect all removals into the HashSet
+        for removal in diff_node.1.diff.removals {
+            removals_set.insert(removal);
         }
     }
+
+    // Remove all links that are in the removals set with a single retain call - O(N)
+    perspective.links.retain(|link| !removals_set.contains(link));
 
     let fn_end = get_now()?.time();
     debug!(

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/snapshots.rs

@@ -1,6 +1,6 @@
 use hdk::prelude::*;
 use perspective_diff_sync_integrity::{
-    LinkExpression, LinkTypes, PerspectiveDiff, PerspectiveDiffEntryReference, Snapshot,
+    LinkExpression, LinkTypes, PerspectiveDiffEntryReference, Snapshot,
 };
 
 use crate::errors::{SocialContextError, SocialContextResult};
@@ -147,20 +147,12 @@ fn handle_parents(
     //Check if entry is already in graph
     if !seen.contains(&search_position.hash) {
         seen.insert(search_position.hash.clone());
-        let diff_entry = get(diff.diff.clone(), GetOptions::network())?
-            .ok_or(SocialContextError::InternalError(
-                "Could not find diff entry for given diff entry reference",
-            ))?
-            .entry()
-            .to_app_option::<PerspectiveDiff>()?
-            .ok_or(SocialContextError::InternalError(
-                "Expected element to contain app entry data",
-            ))?;
-
-        for addition in diff_entry.additions.iter() {
+
+        // Access diff data directly from the entry
+        for addition in diff.diff.additions.iter() {
             all_additions.insert(addition.clone());
         }
-        for removal in diff_entry.removals.iter() {
+        for removal in diff.diff.removals.iter() {
             all_removals.insert(removal.clone());
         }
 

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/topo_sort.rs

@@ -108,36 +108,40 @@ pub fn topo_sort_diff_references(
 #[cfg(test)]
 mod tests {
     use super::topo_sort_diff_references;
+    use crate::errors::SocialContextResult;
     use hdk::prelude::*;
-    use perspective_diff_sync_integrity::PerspectiveDiffEntryReference;
+    use perspective_diff_sync_integrity::{PerspectiveDiff, PerspectiveDiffEntryReference};
 
     #[test]
-    fn test_topo_sort_diff_references() {
+    fn test_topo_sort() -> SocialContextResult<()> {
         let h1 = HoloHash::<holo_hash::hash_type::Action>::from_raw_36(vec![1; 36]);
         let h2 = HoloHash::<holo_hash::hash_type::Action>::from_raw_36(vec![2; 36]);
         let h3 = HoloHash::<holo_hash::hash_type::Action>::from_raw_36(vec![3; 36]);
         let h4 = HoloHash::<holo_hash::hash_type::Action>::from_raw_36(vec![4; 36]);
 
-        let r1 = PerspectiveDiffEntryReference::new(h1.clone(), Some(vec![h2.clone(), h3.clone()]));
-        let r2 = PerspectiveDiffEntryReference::new(h2.clone(), Some(vec![h4.clone()]));
-        let r3 = PerspectiveDiffEntryReference::new(h3.clone(), Some(vec![h4.clone()]));
-        let r4 = PerspectiveDiffEntryReference::new(h4.clone(), None);
+        let r1 = PerspectiveDiffEntryReference::new(PerspectiveDiff::new(), Some(vec![h2.clone(), h3.clone()]));
+        let r2 = PerspectiveDiffEntryReference::new(PerspectiveDiff::new(), Some(vec![h4.clone()]));
+        let r3 = PerspectiveDiffEntryReference::new(PerspectiveDiff::new(), Some(vec![h4.clone()]));
+        let r4 = PerspectiveDiffEntryReference::new(PerspectiveDiff::new(), None);
 
-        let e1 = (h1, r1);
-        let e2 = (h2, r2);
-        let e3 = (h3, r3);
-        let e4 = (h4, r4);
+        let example_arr = vec![(h1, r1), (h2, r2), (h3, r3), (h4, r4)];
 
-        assert_eq!(e1.0, e1.1.diff);
-        assert_eq!(e2.0, e2.1.diff);
-        assert_eq!(e3.0, e3.1.diff);
-        assert_eq!(e4.0, e4.1.diff);
-
-        let test_vec = vec![e1.clone(), e2.clone(), e3.clone(), e4.clone()];
-        let expected = vec![e4, e3, e2, e1];
-
-        let result = topo_sort_diff_references(&test_vec).expect("topo sort to not error");
-
-        assert_eq!(result, expected);
+        let sorted = topo_sort_diff_references(&example_arr)?;
+        assert_eq!(sorted.len(), 4);
+
+        // Check that all diffs are empty (since we created them that way)
+        for item in &sorted {
+            assert!(item.1.diff.additions.is_empty() && item.1.diff.removals.is_empty());
+        }
+
+        // Find the item with no parents (should be first in topo order)
+        let orphan_count = sorted.iter().filter(|item| item.1.parents.is_none()).count();
+        assert_eq!(orphan_count, 1, "Should have exactly one orphan node");
+
+        // Find the item with parents
+        let parent_count = sorted.iter().filter(|item| item.1.parents.is_some()).count();
+        assert_eq!(parent_count, 3, "Should have exactly three nodes with parents");
+
+        Ok(())
     }
 }

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/link_adapter/workspace.rs

@@ -153,22 +153,28 @@ impl Workspace {
                 } else {
                     let mut snapshot = snapshot.unwrap();
 
+                    // Process chunked diffs from snapshot
                     let mut last_diff = None;
                     for i in 0..snapshot.diff_chunks.len() {
-                        let diff_chunk = &snapshot.diff_chunks[i];
+                        let diff_chunk_hash = &snapshot.diff_chunks[i];
+
+                        // Retrieve the actual chunked diff entry
+                        let chunked_diff_entry = Self::get_p_diff_reference::<Retriever>(diff_chunk_hash.clone())?;
+
                         self.entry_map.insert(
-                            diff_chunk.clone(),
-                            PerspectiveDiffEntryReference::new(
-                                diff_chunk.clone(),
-                                last_diff.clone(),
-                            ),
+                            diff_chunk_hash.clone(),
+                            chunked_diff_entry,
                         );
-                        last_diff = Some(vec![diff_chunk.clone()]);
+                        last_diff = Some(vec![diff_chunk_hash.clone()]);
                     }
 
+                    // Insert the current snapshot reference entry with empty diff and link to last chunk
                     self.entry_map.insert(
                         current_hash.clone(),
-                        PerspectiveDiffEntryReference::new(current_diff.diff, last_diff.clone()),
+                        PerspectiveDiffEntryReference::new(
+                            PerspectiveDiff::new(), // empty diff for snapshot reference itself
+                            last_diff.clone(),
+                        ),
                     );
 
                     snapshot_seen.append(&mut snapshot.included_diffs);
@@ -373,8 +379,11 @@ impl Workspace {
             other_mut.found_ancestors.get_mut().push(NULL_NODE());
         };
         if self.diffs.get(&NULL_NODE()).is_none() {
-            let current_diff = PerspectiveDiffEntryReference::new(NULL_NODE(), None);
-            self.diffs.insert(NULL_NODE(), current_diff.clone());
+            let current_diff = PerspectiveDiffEntryReference::new(
+                PerspectiveDiff::new(), // Empty diff for NULL_NODE
+                None
+            );
+            self.diffs.insert(NULL_NODE(), current_diff);
         };
 
         let mut set = if let Some(nodes_back_links) = self.back_links.get(&NULL_NODE()) {
@@ -767,12 +776,12 @@ impl Workspace {
             removals: vec![],
         };
         for (_key, value) in self.entry_map.iter() {
-            if value.diff == NULL_NODE() {
+            if _key == &NULL_NODE() {
                 continue;
             }
-            let diff_entry = Retriever::get::<PerspectiveDiff>(value.diff.clone())?;
-            out.additions.append(&mut diff_entry.additions.clone());
-            out.removals.append(&mut diff_entry.removals.clone());
+            // Access diff data directly from the embedded field
+            out.additions.append(&mut value.diff.additions.clone());
+            out.removals.append(&mut value.diff.removals.clone());
         }
 
         //let fn_end = get_now()?.time();

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/retriever/holochain.rs

@@ -67,7 +67,7 @@ impl PerspectiveDiffRetreiver for HolochainRetreiver {
         let query = query(
             QueryFilter::new()
                 .entry_type(EntryType::App(AppEntryDef {
-                    entry_index: 4.into(),
+                    entry_index: 3.into(),
                     zome_index: 0.into(),
                     visibility: EntryVisibility::Private,
                 }))

bootstrap-languages/p-diff-sync/hc-dna/zomes/perspective_diff_sync/src/retriever/mock.rs

@@ -260,7 +260,13 @@ impl MockPerspectiveGraph {
             } else {
                 None
             };
-            let mocked_diff = PerspectiveDiffEntryReference::new(mocked_hash.clone(), parents);
+            let mocked_diff = PerspectiveDiffEntryReference::new(
+                PerspectiveDiff {
+                    additions: vec![create_link_expression(&mocked_hash.to_string(), &mocked_hash.to_string())],
+                    removals: vec![],
+                },
+                parents
+            );
             let sb = mocked_diff
                 .try_into()
                 .expect("Could not create serialized bytes for mocked_diff");
@@ -320,27 +326,16 @@ impl MockPerspectiveGraph {
                         removals: vec![],
                     };
 
-                    //Create a mock hash for the fake diff
-                    let ref_sb = SerializedBytes::try_from(diff.clone())?;
-                    let mut hasher = Sha256::new();
-                    hasher.update(ref_sb.bytes());
-                    let mut result = hasher.finalize().as_slice().to_owned();
-                    result.append(&mut vec![0xdb, 0xdb, 0xdb, 0xdb]);
-                    let diff_hash = ActionHash::from_raw_36(result);
-
-                    //Create the diff reference
+                    //Create the diff reference with embedded diff data
                     let diff_ref = PerspectiveDiffEntryReference::new(
-                        diff_hash.clone(),
+                        diff,
                         parents.get(ref_hash).as_ref().cloned().cloned(),
                     );
-                    //Insert the diff reference into the map
+                    //Insert only the diff reference into the map at the node hash
                     let diff_ref_sb = diff_ref
                         .try_into()
                         .expect("Could not create serialized bytes for mocked_diff");
                     graph.graph_map.insert(ref_hash.clone(), diff_ref_sb);
-
-                    //Insert the diff into the map
-                    graph.graph_map.insert(diff_hash, ref_sb);
                 }
 
                 Ok(graph)
@@ -422,8 +417,8 @@ fn can_create_graph_from_dot() {
     }";
 
     let graph = MockPerspectiveGraph::from_dot(dot).expect("from_dot not to return error");
-    //26 since there is a diffref & diff for each node
-    assert_eq!(graph.graph_map.keys().len(), 26);
+    //13 since we only create PerspectiveDiffEntryReference entries, not separate PerspectiveDiff entries
+    assert_eq!(graph.graph_map.keys().len(), 13);
 
     let node_12 = node_id_hash(&dot_structures::Id::Plain(String::from("12")));
     let node_11 = node_id_hash(&dot_structures::Id::Plain(String::from("11")));
@@ -519,12 +514,15 @@ fn can_get_and_create_mocked_holochain_objects() {
     use perspective_diff_sync_integrity::{
         EntryTypes, PerspectiveDiff, PerspectiveDiffEntryReference,
     };
-    let commit = MockPerspectiveGraph::create_entry(EntryTypes::PerspectiveDiff(PerspectiveDiff {
-        additions: vec![],
-        removals: vec![],
-    }));
+    let commit = MockPerspectiveGraph::create_entry(EntryTypes::PerspectiveDiffEntryReference(PerspectiveDiffEntryReference::new(
+        PerspectiveDiff {
+            additions: vec![create_link_expression("test", "test")],
+            removals: vec![],
+        },
+        None,
+    )));
     assert!(commit.is_ok());
 
-    let get_commit = MockPerspectiveGraph::get::<PerspectiveDiff>(commit.unwrap());
+    let get_commit = MockPerspectiveGraph::get::<PerspectiveDiffEntryReference>(commit.unwrap());
     assert!(get_commit.is_ok());
 }