secrets manager
git clone https://git.lucas.co/cce-secrets.git
src/bin/cce-keyring-sync/sync.rs (21.7K)
1 //! The three-way merge against an `Interchange` — the 1Password path.
2 //!
3 //! Same table as KEYRING-SYNC.md's, keyed by the interchange's item id
4 //! (`op-item` on the keyring side). Unlike the kdbx merge this replaced,
5 //! there is no file, so nothing is batched — every remote write is
6 //! one `op` call, and the first remote failure stops the apply loop with the
7 //! base snapshot kept for everything not yet applied, so the next run
8 //! re-plans from the same place. Conflict losers need no History push:
9 //! 1Password records item history on every edit.
10 //!
11 //! Change detection on the remote side is by `updated_at`: an entry whose
12 //! timestamp still equals the base's is unchanged and never fetched, so a
13 //! quiet tick is one `op item list` and no secrets.
14
15 use std::collections::HashMap;
16
17 use secret_service::{EncryptionType, SecretService};
18
19 use crate::adopt::{OP_ITEM_ATTR, OP_VAULT_ATTR};
20 use crate::op::{Interchange, RemoteEntry};
21 use crate::{journal_append, keyring_get, now_unix, take_lock, write_state, EntryState, KrEntry, State, APP};
22
23 /// Allowed clock skew before "newer" means anything (the keyring's
24 /// `Modified` is local time; the remote's is the server's).
25 pub const SKEW_TOLERANCE_SECS: i64 = 3;
26
27 /// Items the remote lists that must never reach the keyring: 1Password's
28 /// own account item carries the Secret Key and account password.
29 pub fn excluded_title(title: &str) -> bool {
30 title.starts_with("1Password Account")
31 }
32
33 /// What one entry needs done.
34 #[derive(Debug, Clone, Copy, PartialEq)]
35 pub enum Plan {
36 ToKeyring,
37 ToRemote,
38 /// Born in the keyring (or resurrected there): create remotely, stamp.
39 CreateRemote,
40 /// Both changed: newer wins, tie to the remote.
41 ConflictRemoteWins,
42 ConflictKeyringWins,
43 DeleteKeyring,
44 RecycleRemote,
45 InSync,
46 /// Gone on both sides: drop the base.
47 Forget,
48 }
49
50 /// One side's view of an entry for planning: its field hash and mtime.
51 #[derive(Debug, Clone, PartialEq)]
52 pub struct Side {
53 pub hash: String,
54 pub time: i64,
55 }
56
57 /// The merge table, pure. `base` is the last-synced hash, if any.
58 pub fn plan(base: Option<&str>, remote: Option<&Side>, keyring: Option<&Side>) -> Plan {
59 let newer_remote = |r: &Side, k: &Side| (r.time - k.time).abs() <= SKEW_TOLERANCE_SECS || r.time >= k.time;
60 match (base, remote, keyring) {
61 (None, Some(_), None) => Plan::ToKeyring,
62 (None, None, Some(_)) => Plan::CreateRemote,
63 (None, Some(r), Some(k)) => {
64 // Stamped but no base (a run died before writing state).
65 if r.hash == k.hash {
66 Plan::InSync
67 } else if newer_remote(r, k) {
68 Plan::ConflictRemoteWins
69 } else {
70 Plan::ConflictKeyringWins
71 }
72 }
73 (Some(b), Some(r), Some(k)) => match (r.hash != b, k.hash != b) {
74 (false, false) => Plan::InSync,
75 (true, false) => Plan::ToKeyring,
76 (false, true) => Plan::ToRemote,
77 (true, true) => {
78 if newer_remote(r, k) {
79 Plan::ConflictRemoteWins
80 } else {
81 Plan::ConflictKeyringWins
82 }
83 }
84 },
85 // Deleted on one side; modification on the other beats deletion.
86 (Some(b), None, Some(k)) => {
87 if k.hash != b {
88 Plan::CreateRemote
89 } else {
90 Plan::DeleteKeyring
91 }
92 }
93 (Some(b), Some(r), None) => {
94 if r.hash != b {
95 Plan::ToKeyring
96 } else {
97 Plan::RecycleRemote
98 }
99 }
100 (Some(_), None, None) | (None, None, None) => Plan::Forget,
101 }
102 }
103
104 fn remote_to_kr(e: &RemoteEntry, modified: u64) -> KrEntry {
105 KrEntry {
106 title: e.title.clone(),
107 username: e.username.clone(),
108 password: e.password.clone(),
109 url: e.url.clone(),
110 notes: e.notes.clone(),
111 group: e.vault.clone(),
112 modified,
113 }
114 }
115
116 fn kr_to_remote(k: &KrEntry, id: &str, vault: &str) -> RemoteEntry {
117 RemoteEntry {
118 id: id.to_string(),
119 vault: vault.to_string(),
120 title: k.title.clone(),
121 username: k.username.clone(),
122 password: k.password.clone(),
123 url: k.url.clone(),
124 notes: k.notes.clone(),
125 updated: 0,
126 updated_raw: String::new(),
127 }
128 }
129
130 fn keyring_attrs<'a>(k: &'a KrEntry, id: &'a str, extra: &'a HashMap<String, String>) -> HashMap<&'a str, &'a str> {
131 // Keep whatever else the item carried (kdbx-uuid, xdg:schema, …).
132 let mut a: HashMap<&str, &str> = extra.iter().map(|(x, y)| (x.as_str(), y.as_str())).collect();
133 a.insert(OP_ITEM_ATTR, id);
134 a.insert(OP_VAULT_ATTR, k.group.as_str());
135 a.insert("UserName", k.username.as_str());
136 a.insert("URL", k.url.as_str());
137 a.insert("Notes", k.notes.as_str());
138 a
139 }
140
141 struct Local<'a> {
142 item: secret_service::Item<'a>,
143 attrs: HashMap<String, String>,
144 entry: KrEntry,
145 }
146
147 /// One merge pass. Always writes the state file on a real run (with
148 /// `last_result` set to the outcome, success or not) unless it could not
149 /// even start. Returns the one-line summary, or the error.
150 pub async fn sync_remote<I: Interchange>(
151 remote: &mut I,
152 state_path: &std::path::Path,
153 state: &mut State,
154 dry_run: bool,
155 ) -> Result<String, String> {
156 let Some(_lock) = take_lock() else {
157 return Err("another cce-keyring-sync is running".into());
158 };
159 if state.backend != "onepassword" {
160 return Err("the sync base is not 1Password's — run `cce-keyring-sync adopt` first".into());
161 }
162 let vault = state.vault.clone();
163
164 let ss = SecretService::connect(EncryptionType::Dh)
165 .await
166 .map_err(|e| format!("Secret Service unavailable: {e}"))?;
167 let hash_key: [u8; 32] = match keyring_get(&ss, "state-hash-key").await {
168 Ok(Some(b)) if b.len() == 32 => b.try_into().unwrap(),
169 _ => return Err("no state hash key — run `cce-keyring-sync adopt` first".into()),
170 };
171 let col = ss.get_default_collection().await.map_err(|e| format!("no default collection: {e}"))?;
172 if col.is_locked().await.unwrap_or(false) && col.unlock().await.is_err() {
173 return Err("collection locked".into());
174 }
175
176 // ---- keyring snapshot ----
177 let mut kr: HashMap<String, Local<'_>> = HashMap::new();
178 let mut born: Vec<Local<'_>> = Vec::new();
179 for item in col.get_all_items().await.map_err(|e| format!("listing collection failed: {e}"))? {
180 let Ok(attrs) = item.get_attributes().await else { continue };
181 if attrs.get("application").map(String::as_str) == Some(APP) {
182 continue;
183 }
184 let id = attrs.get(OP_ITEM_ATTR).cloned();
185 if id.is_none() && !attrs.contains_key("UserName") && !attrs.contains_key("kdbx-uuid") {
186 continue; // some other app's item — never ours to sync
187 }
188 let entry = KrEntry {
189 title: item.get_label().await.unwrap_or_default(),
190 username: attrs.get("UserName").cloned().unwrap_or_default(),
191 password: String::from_utf8_lossy(&item.get_secret().await.unwrap_or_default()).into_owned(),
192 url: attrs.get("URL").cloned().unwrap_or_default(),
193 notes: attrs.get("Notes").cloned().unwrap_or_default(),
194 group: attrs.get(OP_VAULT_ATTR).cloned().unwrap_or_else(|| vault.clone()),
195 modified: item.get_modified().await.unwrap_or(0),
196 };
197 let local = Local { item, attrs, entry };
198 match id {
199 Some(id) => {
200 // Two items with one stamp (a tool that re-created rather than
201 // edited): the newer one is the person's latest word.
202 let newer = kr.get(&id).is_none_or(|old| local.entry.modified >= old.entry.modified);
203 if newer {
204 kr.insert(id, local);
205 }
206 }
207 None => born.push(local),
208 }
209 }
210
211 // ---- remote snapshot: the list, then fetches only where needed ----
212 let summaries = remote.list().await?;
213 let mut rs: HashMap<String, crate::op::RemoteSummary> = HashMap::new();
214 for s in summaries {
215 if excluded_title(&s.title) {
216 continue;
217 }
218 rs.insert(s.id.clone(), s);
219 }
220 let mut fetched: HashMap<String, RemoteEntry> = HashMap::new();
221 let mut fetches = 0usize;
222
223 let mut ids: Vec<String> = state.entries.keys().chain(rs.keys()).chain(kr.keys()).cloned().collect();
224 ids.sort();
225 ids.dedup();
226
227 // ---- plan ----
228 let mut plans: Vec<(String, Plan)> = Vec::new();
229 for id in &ids {
230 let base = state.entries.get(id);
231 let k_side = kr.get(id).map(|l| Side { hash: l.entry.hash(&hash_key), time: l.entry.modified as i64 });
232 let r_side = match rs.get(id) {
233 None => None,
234 Some(s) => {
235 let unchanged = base.is_some_and(|b| !b.op_updated_at.is_empty() && b.op_updated_at == s.updated_raw);
236 if unchanged {
237 Some(Side { hash: base.unwrap().h.clone(), time: s.updated })
238 } else {
239 let e = remote.fetch(id).await?;
240 fetches += 1;
241 let h = remote_to_kr(&e, 0).hash(&hash_key);
242 fetched.insert(id.clone(), e);
243 Some(Side { hash: h, time: s.updated })
244 }
245 }
246 };
247 plans.push((id.clone(), plan(base.map(|b| b.h.as_str()), r_side.as_ref(), k_side.as_ref())));
248 }
249
250 // ---- report ----
251 let title_of = |id: &str| -> String {
252 rs.get(id)
253 .map(|s| s.title.clone())
254 .or_else(|| kr.get(id).map(|l| l.entry.title.clone()))
255 .unwrap_or_else(|| id.to_string())
256 };
257 let mut journal = String::new();
258 let mut counts: HashMap<&'static str, usize> = HashMap::new();
259 for (id, p) in &plans {
260 let verb = match p {
261 Plan::ToKeyring => "1Password -> keyring",
262 Plan::ToRemote => "keyring -> 1Password",
263 Plan::CreateRemote => "create in 1Password",
264 Plan::ConflictRemoteWins => "CONFLICT: 1Password wins (loser in item history)",
265 Plan::ConflictKeyringWins => "CONFLICT: keyring wins (loser in item history)",
266 Plan::DeleteKeyring => "delete from keyring",
267 Plan::RecycleRemote => "archive in 1Password",
268 Plan::InSync | Plan::Forget => continue,
269 };
270 *counts
271 .entry(match p {
272 Plan::ToKeyring | Plan::ConflictRemoteWins => "to-keyring",
273 Plan::ToRemote | Plan::ConflictKeyringWins => "to-remote",
274 Plan::CreateRemote => "created",
275 Plan::DeleteKeyring => "deleted",
276 Plan::RecycleRemote => "archived",
277 _ => unreachable!(),
278 })
279 .or_default() += 1;
280 println!(" {verb}: {}", title_of(id));
281 journal.push_str(&format!("{} sync {verb}: {}\n", now_unix(), title_of(id)));
282 }
283 for l in &born {
284 println!(" create in 1Password: {}", l.entry.title);
285 journal.push_str(&format!("{} sync create in 1Password: {}\n", now_unix(), l.entry.title));
286 *counts.entry("created").or_default() += 1;
287 }
288 let c = |k: &str| counts.get(k).copied().unwrap_or(0);
289 let quiet = plans.iter().all(|(_, p)| matches!(p, Plan::InSync | Plan::Forget)) && born.is_empty();
290 let summary = if quiet {
291 "in sync".to_string()
292 } else {
293 format!(
294 "synced: {} -> keyring, {} -> 1Password, {} created, {} deleted, {} archived",
295 c("to-keyring"),
296 c("to-remote"),
297 c("created"),
298 c("deleted"),
299 c("archived")
300 )
301 };
302 if dry_run {
303 println!("{summary} (dry run — nothing changed; {fetches} fetched)");
304 return Ok(summary);
305 }
306
307 // ---- apply ----
308 // `next` starts as the old base and is rewritten entry by entry, so a
309 // remote failure mid-way leaves untouched entries with their old base.
310 let mut next: HashMap<String, EntryState> = state.entries.drain().collect();
311 let mut failure: Option<String> = None;
312 let mut applied = 0usize;
313
314 let snapshot = |k: &KrEntry, updated_raw: String, keyring_modified: u64| EntryState {
315 h: k.hash(&hash_key),
316 keyring_modified,
317 op_updated_at: updated_raw,
318 };
319 'apply: for (id, p) in &plans {
320 match p {
321 Plan::InSync => {
322 // Keep the base timestamp on the list's value: it was unknown
323 // after adopt (the drift marker), and the server may stamp a
324 // write a second later than the reply we recorded. Either way
325 // the entry would be fetched every tick until this catches up.
326 if let (Some(s), Some(b)) = (rs.get(id), next.get_mut(id)) {
327 if b.op_updated_at != s.updated_raw {
328 b.op_updated_at = s.updated_raw.clone();
329 }
330 }
331 }
332 Plan::Forget => {
333 next.remove(id);
334 }
335 Plan::ToKeyring | Plan::ConflictRemoteWins => {
336 let e = match fetched.get(id) {
337 Some(e) => e.clone(),
338 None => match remote.fetch(id).await {
339 Ok(e) => {
340 fetches += 1;
341 e
342 }
343 Err(err) => {
344 failure = Some(err);
345 break 'apply;
346 }
347 },
348 };
349 let k = remote_to_kr(&e, 0);
350 let empty = HashMap::new();
351 let modified = match kr.get(id) {
352 Some(l) => {
353 let attrs = keyring_attrs(&k, id, &l.attrs);
354 let r = async {
355 l.item.set_label(&k.title).await?;
356 l.item.set_attributes(attrs).await?;
357 l.item.set_secret(k.password.as_bytes(), "text/plain").await?;
358 l.item.get_modified().await
359 }
360 .await;
361 match r {
362 Ok(m) => m,
363 Err(err) => {
364 eprintln!(" keyring write failed for {}: {err}", k.title);
365 continue;
366 }
367 }
368 }
369 None => {
370 let attrs = keyring_attrs(&k, id, &empty);
371 match col.create_item(&k.title, attrs, k.password.as_bytes(), true, "text/plain").await {
372 Ok(item) => item.get_modified().await.unwrap_or(now_unix() as u64),
373 Err(err) => {
374 eprintln!(" keyring create failed for {}: {err}", k.title);
375 continue;
376 }
377 }
378 }
379 };
380 next.insert(id.clone(), snapshot(&k, e.updated_raw.clone(), modified));
381 applied += 1;
382 }
383 Plan::ToRemote | Plan::ConflictKeyringWins => {
384 let l = &kr[id];
385 let e = kr_to_remote(&l.entry, id, &l.entry.group);
386 match remote.update(&e).await {
387 Ok(updated_raw) => {
388 next.insert(id.clone(), snapshot(&l.entry, updated_raw, l.entry.modified));
389 applied += 1;
390 }
391 Err(err) => {
392 failure = Some(err);
393 break 'apply;
394 }
395 }
396 }
397 Plan::CreateRemote => {
398 // A keyring entry whose stamp points at nothing any more
399 // (archived remotely, edited locally): create afresh, restamp.
400 let l = &kr[id];
401 let mut e = kr_to_remote(&l.entry, "", &vault);
402 e.vault = vault.clone();
403 match remote.create(&e).await {
404 Ok((new_id, updated_raw)) => {
405 let mut k = l.entry.clone();
406 k.group = vault.clone();
407 let attrs = keyring_attrs(&k, &new_id, &l.attrs);
408 let modified = match async {
409 l.item.set_attributes(attrs).await?;
410 l.item.get_modified().await
411 }
412 .await
413 {
414 Ok(m) => m,
415 Err(err) => {
416 eprintln!(" could not restamp {}: {err}", k.title);
417 l.entry.modified
418 }
419 };
420 next.remove(id);
421 next.insert(new_id, snapshot(&k, updated_raw, modified));
422 applied += 1;
423 }
424 Err(err) => {
425 failure = Some(err);
426 break 'apply;
427 }
428 }
429 }
430 Plan::DeleteKeyring => {
431 let l = &kr[id];
432 match l.item.delete().await {
433 Ok(()) => {
434 next.remove(id);
435 applied += 1;
436 }
437 Err(err) => eprintln!(" keyring delete failed for {}: {err}", l.entry.title),
438 }
439 }
440 Plan::RecycleRemote => match remote.recycle(id).await {
441 Ok(()) => {
442 next.remove(id);
443 applied += 1;
444 }
445 Err(err) => {
446 failure = Some(err);
447 break 'apply;
448 }
449 },
450 }
451 }
452 if failure.is_none() {
453 for l in &born {
454 let mut k = l.entry.clone();
455 k.group = vault.clone();
456 let e = kr_to_remote(&k, "", &vault);
457 match remote.create(&e).await {
458 Ok((new_id, updated_raw)) => {
459 let attrs = keyring_attrs(&k, &new_id, &l.attrs);
460 let modified = match async {
461 l.item.set_attributes(attrs).await?;
462 l.item.get_modified().await
463 }
464 .await
465 {
466 Ok(m) => m,
467 Err(err) => {
468 eprintln!(" could not stamp {}: {err}", k.title);
469 l.entry.modified
470 }
471 };
472 next.insert(new_id, snapshot(&k, updated_raw, modified));
473 applied += 1;
474 }
475 Err(err) => {
476 failure = Some(err);
477 break;
478 }
479 }
480 }
481 }
482
483 state.entries = next;
484 state.last_run = now_unix();
485 let result = match failure {
486 None => Ok(summary.clone()),
487 Some(err) => {
488 let changes = plans.iter().filter(|(_, p)| !matches!(p, Plan::InSync | Plan::Forget)).count() + born.len();
489 Err(format!("{err} (after {applied} of {changes} changes; the rest retry next run)"))
490 }
491 };
492 state.last_result = match &result {
493 Ok(s) => s.clone(),
494 Err(e) => format!("failed: {e}"),
495 };
496 write_state(state_path, state);
497 if !journal.is_empty() {
498 journal_append(&journal);
499 }
500 if let Err(e) = &result {
501 journal_append(&format!("{} sync FAILED: {e}\n", now_unix()));
502 }
503 println!("{} ({fetches} fetched)", state.last_result);
504 result
505 }
506
507 #[cfg(test)]
508 mod tests {
509 use super::*;
510
511 fn side(h: &str, t: i64) -> Side {
512 Side { hash: h.into(), time: t }
513 }
514
515 #[test]
516 fn the_merge_table() {
517 let b = Some("B");
518 assert_eq!(plan(None, Some(&side("R", 0)), None), Plan::ToKeyring);
519 assert_eq!(plan(None, None, Some(&side("K", 0))), Plan::CreateRemote);
520 assert_eq!(plan(b, Some(&side("B", 0)), Some(&side("B", 0))), Plan::InSync);
521 assert_eq!(plan(b, Some(&side("R", 0)), Some(&side("B", 0))), Plan::ToKeyring);
522 assert_eq!(plan(b, Some(&side("B", 0)), Some(&side("K", 0))), Plan::ToRemote);
523 assert_eq!(plan(b, None, Some(&side("B", 0))), Plan::DeleteKeyring);
524 assert_eq!(plan(b, None, Some(&side("K", 0))), Plan::CreateRemote, "modification beats deletion");
525 assert_eq!(plan(b, Some(&side("B", 0)), None), Plan::RecycleRemote);
526 assert_eq!(plan(b, Some(&side("R", 0)), None), Plan::ToKeyring, "modification beats deletion");
527 assert_eq!(plan(b, None, None), Plan::Forget);
528 assert_eq!(plan(None, None, None), Plan::Forget);
529 }
530
531 #[test]
532 fn conflicts_go_to_the_newer_side_and_ties_to_the_remote() {
533 let b = Some("B");
534 assert_eq!(plan(b, Some(&side("R", 100)), Some(&side("K", 50))), Plan::ConflictRemoteWins);
535 assert_eq!(plan(b, Some(&side("R", 50)), Some(&side("K", 100))), Plan::ConflictKeyringWins);
536 assert_eq!(plan(b, Some(&side("R", 98)), Some(&side("K", 100))), Plan::ConflictRemoteWins, "inside the skew tolerance is a tie");
537 assert_eq!(plan(b, Some(&side("R", 100)), Some(&side("K", 100))), Plan::ConflictRemoteWins);
538 }
539
540 #[test]
541 fn a_stamped_entry_without_a_base_is_reconciled_by_hash() {
542 assert_eq!(plan(None, Some(&side("X", 0)), Some(&side("X", 0))), Plan::InSync);
543 assert_eq!(plan(None, Some(&side("R", 10)), Some(&side("K", 0))), Plan::ConflictRemoteWins);
544 }
545
546 #[test]
547 fn the_account_item_is_excluded() {
548 assert!(excluded_title("1Password Account (alice)"));
549 assert!(!excluded_title("Account at 1Password"));
550 }
551 }