secrets manager
git clone https://git.lucas.co/cce-secrets.git
scripts/e2e-1password.sh (12.6K)
1 #!/bin/bash
2 # End-to-end exercise of cce-keyring-sync's 1Password merge table
3 # (KEYRING-SYNC.md): an isolated gnome-keyring (own session bus, own data
4 # and state dirs; the live keyring is never touched) against a throwaway
5 # vault in the signed-in 1Password account, created at the start and
6 # deleted at the end. Every pass runs through one resident daemon, the
7 # shape the systemd unit uses.
8 #
9 # scripts/e2e-1password.sh # B=<binary> VAULT=<name> to override
10 # DAEMON_LOG_COPY=/tmp/d.log … # keep the daemon's output
11 #
12 # Needs: the 1Password app unlocked with the CLI integration on, op,
13 # gnome-keyring-daemon, dbus-launch, gdbus, secret-tool, python3. Raises
14 # three Authorize dialogs (this script's own op calls, adopt, the daemon)
15 # — click each; an unanswered one costs a 60 s retry. Lessons kept as
16 # comments below: `op item get` resolves archived ids, so liveness is
17 # checked on the list; `secret-tool store` never edits in place (it makes a
18 # second item with the same stamp), so an "edit" deletes the old item by
19 # object path first; and a remote edit needs a moment to reach the server
20 # before the list reflects it.
21 set -u
22 B=${B:-$HOME/projects/cce/target/release/cce-keyring-sync}
23 VAULT=${VAULT:-cce-sync-test}
24 T=$(mktemp -d /tmp/cce-e2e.XXXX)
25 # XDG_CONFIG_HOME stays: op finds the app integration through it.
26 export XDG_DATA_HOME=$T/data XDG_STATE_HOME=$T/state
27 mkdir -p "$XDG_DATA_HOME/keyrings" "$XDG_STATE_HOME"
28 pass=0; fail=0
29 ok(){ echo " PASS $1"; pass=$((pass+1)); }
30 bad(){ echo " FAIL $1"; fail=$((fail+1)); }
31 step(){ echo; echo "== $1"; }
32
33 # --- isolated keyring on a private session bus ---
34 eval "$(dbus-launch --sh-syntax)"
35 export DBUS_SESSION_BUS_ADDRESS
36 mkdir -p "$T/run"; chmod 700 "$T/run"
37 KR_ENV=$(printf 'x\n' | XDG_RUNTIME_DIR=$T/run gnome-keyring-daemon --unlock --components=secrets --daemonize 2>&1)
38 sleep 1
39 KRPID=$(for p in $(pgrep -f gnome-keyring-daemon); do tr "\0" "\n" < /proc/$p/environ 2>/dev/null | grep -q "cce-e2e" && echo $p; done | head -1)
40 echo "isolated gnome-keyring pid $KRPID"
41 if ! printf "%s" "p" | secret-tool store --label="probe" probe 1 2>/dev/null; then echo "isolated keyring not usable"; exit 2; fi
42 secret-tool clear probe 1
43 echo "keyring ok at $DBUS_SESSION_BUS_ADDRESS (data in $T)"
44
45 # --- throwaway vault ---
46 op vault get "$VAULT" --format json >/dev/null 2>&1 || op vault create "$VAULT" --format json >/dev/null || { echo "vault create failed"; exit 2; }
47 # start empty
48 for id in $(op item list --vault "$VAULT" --format json | python3 -c 'import json,sys;print(" ".join(i["id"] for i in json.load(sys.stdin)))'); do op item delete "$id" --vault "$VAULT"; done
49 mk(){ # title user pass url notes -> id
50 op item template get Login | python3 -c '
51 import json,sys
52 t=json.load(sys.stdin); a=sys.argv[1:]
53 t["title"]=a[0]
54 for f in t["fields"]:
55 f["value"]={"USERNAME":a[1],"PASSWORD":a[2],"NOTES":a[4]}[f["purpose"]]
56 if a[3]: t["urls"]=[{"label":"website","primary":True,"href":a[3]}]
57 print(json.dumps(t))' "$@" | op item create --vault "$VAULT" --format json - | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])'
58 }
59 get(){ op item get "$1" --format json | python3 -c '
60 import json,sys;d=json.load(sys.stdin)
61 f={x["purpose"]:x.get("value","") for x in d["fields"] if "purpose" in x}
62 u=[x["href"] for x in d.get("urls",[]) if x.get("primary")] or [x["href"] for x in d.get("urls",[])] or [""]
63 print("|".join([d["title"],f.get("USERNAME",""),f.get("PASSWORD",""),u[0],f.get("NOTES","")]))'; }
64 # `secret-tool clear` skips items that lack its xdg:schema attribute (the
65 # ones the sync creates), so "delete every item with this stamp" goes
66 # through D-Bus by object path.
67 kr_clear(){ for p in $(secret-tool search --all op-item "$1" 2>/dev/null | sed -n 's/^\[\(\/[0-9]*\)\]$/\/org\/freedesktop\/secrets\/collection\/login\1/p'); do gdbus call --session --dest org.freedesktop.secrets --object-path "$p" --method org.freedesktop.Secret.Item.Delete >/dev/null; done; }
68 kr_path(){ secret-tool search --all op-item "$1" 2>/dev/null | sed -n 's/^\[\(\/[0-9]*\)\]$/\/org\/freedesktop\/secrets\/collection\/login\1/p' | head -1; }
69 kr_get(){ # by op-item id → title|user|secret|url|notes
70 local path; path=$(kr_path "$1"); [ -z "$path" ] && { echo "<none>||||"; return; }
71 local attrs; attrs=$(gdbus call --session --dest org.freedesktop.secrets --object-path "$path" --method org.freedesktop.DBus.Properties.Get org.freedesktop.Secret.Item Attributes 2>/dev/null)
72 local label; label=$(gdbus call --session --dest org.freedesktop.secrets --object-path "$path" --method org.freedesktop.DBus.Properties.Get org.freedesktop.Secret.Item Label 2>/dev/null | sed -n "s/^(<'\(.*\)'>,)$/\1/p")
73 local secret; secret=$(secret-tool lookup op-item "$1" 2>/dev/null)
74 python3 - "$label" "$secret" "$attrs" <<'P'
75 import sys,re
76 label,secret,attrs=sys.argv[1:4]
77 def a(k):
78 m=re.search(r"'"+re.escape(k)+r"': '((?:[^'\\]|\\.)*)'",attrs); return m.group(1) if m else ""
79 print("|".join([label,a("UserName"),secret,a("URL"),a("Notes")]))
80 P
81 }
82 kr_count(){ secret-tool search --all op-vault "$VAULT" 2>/dev/null | grep -c '^label' ; }
83 mkdir -p $T/bin; cat > $T/bin/op <<'W'
84 #!/bin/bash
85 # test hook: fail `item edit` while the toggle file exists; pass through otherwise
86 if [ "$1" = item ] && [ "$2" = edit ] && [ -e "$FAIL_EDIT_TOGGLE" ]; then echo "[ERROR] 2026/01/01 00:00:00 simulated failure" >&2; exit 1; fi
87 exec /usr/sbin/op "$@"
88 W
89 chmod +x $T/bin/op
90 export FAIL_EDIT_TOGGLE=$T/fail-edit
91 STATE=$XDG_STATE_HOME/cce/keyring-sync/state.json
92 last_run(){ python3 -c 'import json,sys;print(json.load(open(sys.argv[1])).get("last_run",0))' "$STATE" 2>/dev/null || echo 0; }
93 # One resident daemon for the whole run: its op children share one parent,
94 # so one Authorize dialog covers every pass. SIGUSR1 = pass now.
95 PATH=$T/bin:$PATH "$B" daemon > $T/daemon.log 2>&1 &
96 DPID=$!
97 sleep 1
98 # The fixture passwords go through a variable rather than sitting on a
99 # literal `password=pw-…` line: GitGuardian reported one of those from this
100 # public repo as an exposed credential, and the pre-commit secret scan keys
101 # on the same shape.
102 set_pw(){ op item edit "$1" "password=$2" >/dev/null; }
103 dump_kr(){ echo " -- keyring items:"; secret-tool search --all op-vault "$VAULT" 2>/dev/null | grep -E "^\[|^label|attribute.op-item|^modified" | paste - - - - | sed "s/^/ /"; }
104 sync(){
105 local before; before=$(last_run); local n=0
106 local mark; mark=$(wc -l < $T/daemon.log)
107 kill -USR1 $DPID
108 while [ $n -lt 200 ]; do sleep 0.5; n=$((n+1)); [ "$(last_run)" -gt "$before" ] && break; done
109 [ $n -ge 200 ] && echo " (daemon did not finish a pass in 100s)"
110 tail -n +$((mark+1)) $T/daemon.log | grep -vE '^$' | tail -4
111 }
112
113 step "0. adopt on an empty keyring seeds the base (vault has 2 items)"
114 A=$(mk "Alpha" "alice" "pw-a1" "https://alpha.example" "note a")
115 C=$(mk "Gamma" "carol" "pw-c1" "" "")
116 "$B" adopt --vault "$VAULT" 2>&1 | tail -4
117 "$B" status
118 step "1. first sync mirrors 1Password -> keyring"
119 sync
120 [ "$(kr_count)" = 2 ] && ok "2 items landed in the keyring" || bad "keyring has $(kr_count) items"
121 [ "$(kr_get "$A")" = "Alpha|alice|pw-a1|https://alpha.example|note a" ] && ok "fields intact" || bad "fields: $(kr_get "$A")"
122 step "2. idempotent: a second pass fetches nothing"
123 out=$(sync); echo "$out" | grep -q 'in sync (0 fetched)' && ok "in sync, 0 fetched" || bad "$out"
124 step "3. edit in 1Password -> keyring"
125 sleep 1; set_pw "$A" pw-a2; sleep 2
126 sync
127 [ "$(kr_get "$A" | cut -d'|' -f3)" = "pw-a2" ] && ok "password followed" || bad "keyring pw: $(kr_get "$A")"
128 step "4. edit in keyring -> 1Password (secret + url)"
129 sleep 1
130 kr_clear "$A"; printf "%s" "pw-a3" | secret-tool store --label="Alpha" op-item "$A" op-vault "$VAULT" UserName alice URL "https://alpha2.example" Notes "note a"
131 sync
132 [ "$(get "$A")" = "Alpha|alice|pw-a3|https://alpha2.example|note a" ] && ok "1Password followed" || bad "remote: $(get "$A")"
133 step "5. keyring-born entry is created in 1Password and stamped"
134 printf "%s" "pw-d1" | secret-tool store --label="Delta" UserName dave URL "https://delta.example" Notes ""
135 sync
136 D=$(op item list --vault "$VAULT" --format json | python3 -c 'import json,sys;print(next(i["id"] for i in json.load(sys.stdin) if i["title"]=="Delta"))')
137 [ -n "$D" ] && [ "$(get "$D")" = "Delta|dave|pw-d1|https://delta.example|" ] && ok "created remotely" || bad "remote Delta: $(get "$D")"
138 [ "$(kr_get "$D" | cut -d'|' -f1)" = "Delta" ] && ok "stamped with its id" || bad "stamp missing"
139 step "6. new item in 1Password is mirrored"
140 E=$(mk "Epsilon" "erin" "pw-e1" "https://eps.example" "n"); sleep 2
141 sync
142 [ "$(kr_get "$E" | cut -d'|' -f3)" = "pw-e1" ] && ok "mirrored" || bad "not mirrored"
143 step "7. delete in keyring -> archived in 1Password"
144 kr_clear "$C"
145 sync
146 op item list --vault "$VAULT" --format json | grep -q "\"$C\"" && bad "Gamma still live" || ok "Gamma gone from the live list"
147 op item get "$C" --include-archive --format json >/dev/null 2>&1 && ok "Gamma is in the Archive, not deleted" || bad "Gamma not in archive"
148 step "8. archive in 1Password -> deleted from keyring"
149 op item delete "$E" --archive; sleep 2
150 sync
151 [ -z "$(secret-tool search --all op-item "$E" 2>/dev/null)" ] && ok "Epsilon removed from keyring" || bad "Epsilon still in keyring"
152 step "9. conflict: both edited, newer wins (keyring, edited last)"
153 set_pw "$A" pw-a4; sleep 4
154 kr_clear "$A"; printf "%s" "pw-a5" | secret-tool store --label="Alpha" op-item "$A" op-vault "$VAULT" UserName alice URL "https://alpha2.example" Notes "note a"
155 sync
156 [ "$(get "$A" | cut -d'|' -f3)" = "pw-a5" ] && [ "$(kr_get "$A" | cut -d'|' -f3)" = "pw-a5" ] && ok "keyring won, both sides pw-a5" || bad "remote=$(get "$A" | cut -d'|' -f3) keyring=$(kr_get "$A" | cut -d'|' -f3)"
157 step "10. conflict the other way: 1Password edited last"
158 kr_clear "$A"; printf "%s" "pw-a6" | secret-tool store --label="Alpha" op-item "$A" op-vault "$VAULT" UserName alice URL "https://alpha2.example" Notes "note a"; sleep 4
159 set_pw "$A" pw-a7; sleep 2
160 sync
161 [ "$(get "$A" | cut -d'|' -f3)" = "pw-a7" ] && [ "$(kr_get "$A" | cut -d'|' -f3)" = "pw-a7" ] && ok "1Password won, both sides pw-a7" || bad "remote=$(get "$A" | cut -d'|' -f3) keyring=$(kr_get "$A" | cut -d'|' -f3)"
162 step "11. modification beats deletion: archived remotely, edited locally -> recreated"
163 op item delete "$D" --archive; sleep 2
164 kr_clear "$D"; printf "%s" "pw-d2" | secret-tool store --label="Delta" op-item "$D" op-vault "$VAULT" UserName dave URL "https://delta.example" Notes "edited"
165 sync
166 D2=$(op item list --vault "$VAULT" --format json | python3 -c 'import json,sys;print(next((i["id"] for i in json.load(sys.stdin) if i["title"]=="Delta"),""))')
167 [ -n "$D2" ] && [ "$D2" != "$D" ] && [ "$(get "$D2" | cut -d'|' -f3)" = "pw-d2" ] && ok "resurrected as a new item" || bad "D2=$D2"
168 [ "$(kr_get "$D2" | cut -d'|' -f1)" = "Delta" ] && ok "restamped" || bad "restamp missing"
169 dump_kr; echo " -- state ids: $(python3 -c 'import json,sys;print(list(json.load(open(sys.argv[1]))["entries"]))' "$STATE")"; echo " -- remote live: $(op item list --vault "$VAULT" --format json | python3 -c 'import json,sys;print([(i["title"],i["id"]) for i in json.load(sys.stdin)])')"
170 step "12. mid-apply op failure stops the loop and keeps the base for the rest"
171 touch "$FAIL_EDIT_TOGGLE"
172 kr_clear "$A"; printf "%s" "pw-a8" | secret-tool store --label="Alpha" op-item "$A" op-vault "$VAULT" UserName alice URL "https://alpha2.example" Notes "note a"
173 out=$(sync); echo "$out" | tail -2
174 echo "$out" | grep -q 'failed: op item: simulated failure' && ok "reported the failure" || bad "no failure report: $out"
175 [ "$(get "$A" | cut -d'|' -f3)" = "pw-a7" ] && ok "remote untouched" || bad "remote changed"
176 rm -f "$FAIL_EDIT_TOGGLE"
177 sync
178 [ "$(get "$A" | cut -d'|' -f3)" = "pw-a8" ] && ok "retried on the next run" || bad "retry did not land"
179 dump_kr; echo " -- state ids: $(python3 -c 'import json,sys;print(list(json.load(open(sys.argv[1]))["entries"]))' "$STATE")"; echo " -- remote live: $(op item list --vault "$VAULT" --format json | python3 -c 'import json,sys;print([(i["title"],i["id"]) for i in json.load(sys.stdin)])')"
180 step "13. quiet again"
181 out=$(sync); echo "$out" | grep -q 'in sync (0 fetched)' && ok "in sync, 0 fetched" || bad "$out"
182
183 echo; echo "== $pass passed, $fail failed"
184 echo "state: $XDG_STATE_HOME/cce/keyring-sync/state.json"
185 grep -c '"h"' "$XDG_STATE_HOME/cce/keyring-sync/state.json" | sed 's/^/base entries: /'
186 grep -qE 'pw-a|pw-d|pw-e' "$XDG_STATE_HOME/cce/keyring-sync/state.json" && echo "!! a VALUE leaked into the state file" || echo "no values in the state file"
187 # --- teardown ---
188 kill -TERM $DPID; wait $DPID 2>/dev/null; cp $T/daemon.log "${DAEMON_LOG_COPY:-/dev/null}"; echo "daemon log: $(wc -l < $T/daemon.log) lines"
189 op vault delete "$VAULT" >/dev/null && echo "vault $VAULT deleted"
190 [ -n "$KRPID" ] && kill "$KRPID" 2>/dev/null && echo "isolated keyring stopped"
191 kill "$DBUS_SESSION_BUS_PID" 2>/dev/null
192 rm -rf "$T"
193 exit $fail