# Bidirectional iCloud Sync Implementation

Complete reference implementation for a conflict-safe bidirectional sync between a headless Linux server and iCloud Drive, using pyicloud.

## The Problem

The default push-only script uploads local files and overwrites iCloud versions when sizes differ. If the user edits on their Mac/iPad, the server will silently destroy those edits on the next sync.

## The Solution

Two-phase sync with conflict detection and no overwrites.

### Full Script

```python
#!/usr/bin/env python3
"""
Bidirectional sync: local Obsidian vault to/from iCloud Drive.

Behavior:
- Uploads NEW local files to iCloud.
- Downloads NEW iCloud files to local (files added from Mac/iPad/phone).
- Skips files that exist on BOTH sides with SAME size.
- Skips files that exist on BOTH sides with DIFFERENT sizes (conflict).

Usage: APPLE_PASSWORD=your_password python3 sync_to_icloud.py
"""

import os
import io
import sys

email = os.environ.get("APPLE_EMAIL")
password = os.environ.get("APPLE_PASSWORD")

if not email or not password:
    print("ERROR: Set APPLE_EMAIL and APPLE_PASSWORD environment variables")
    sys.exit(1)

from pyicloud import PyiCloudService

class NamedBytesIO(io.BytesIO):
    def __init__(self, initial_bytes, name):
        super().__init__(initial_bytes)
        self.name = name

print(f"Authenticating with Apple ID: {email}")
api = PyiCloudService(email, password)

if api.requires_2fa:
    print("ERROR: 2FA required. Session expired.")
    sys.exit(1)

print("Login OK")

drive = api.drive
root = drive.root
obsidian = root["Obsidian"]
campaign = obsidian["Pirate Campain"]

local_root = os.path.expanduser("~/obsidian-vault/Pirate-Campain")

# Build a map of existing iCloud files and dirs
print("\nScanning iCloud...")
icloud_files = {}
icloud_dirs = {}

def scan_icloud(node, path=""):
    children = node.get_children()
    for child in children:
        child_path = f"{path}/{child.name}" if path else child.name
        if child.type == "folder" or child.type == "app_library":
            icloud_dirs[child_path] = child
            scan_icloud(child, child_path)
        else:
            icloud_files[child_path] = child

scan_icloud(campaign)
print(f"Found {len(icloud_files)} files, {len(icloud_dirs)} dirs on iCloud")

# Cache for newly created directories (avoids iCloud lag)
dir_cache = {}

def get_or_create_dir(parent, dirname, rel_path):
    """Get a directory node, creating it if needed. Cache results."""
    cache_key = rel_path if rel_path else dirname

    if cache_key in dir_cache:
        node = dir_cache[cache_key]
        if callable(getattr(node, 'upload', None)) and not isinstance(node, dict):
            return node
        del dir_cache[cache_key]

    if rel_path in icloud_dirs:
        node = icloud_dirs[rel_path]
        if callable(getattr(node, 'upload', None)) and not isinstance(node, dict):
            dir_cache[cache_key] = node
            return node

    def find_child_node(p, name):
        try:
            for child in p.get_children():
                if child.name == name and (child.type == "folder" or child.type == "app_library"):
                    return child
        except Exception:
            pass
        return None

    existing = find_child_node(parent, dirname)
    if existing is not None:
        dir_cache[cache_key] = existing
        return existing

    try:
        result = parent.mkdir(dirname)
    except Exception as e:
        existing = find_child_node(parent, dirname)
        if existing is not None:
            dir_cache[cache_key] = existing
            return existing
        print(f"  [DIR! ] {rel_path} — {e}")
        return None

    if isinstance(result, dict) or not callable(getattr(result, 'upload', None)):
        new_dir = find_child_node(parent, dirname)
        if new_dir is not None:
            dir_cache[cache_key] = new_dir
            return new_dir
        print(f"  [DIR! ] {rel_path} — created dir but could not resolve node")
        return None

    dir_cache[cache_key] = result
    return result

# Phase 1: Upload local files that are new or unchanged on iCloud.
print("\nPhase 1: Uploading new local files to iCloud...")
uploaded = 0
skipped_conflict = 0
skipped_unchanged = 0
errors = 0

for dirpath, dirnames, filenames in os.walk(local_root):
    rel_dir = os.path.relpath(dirpath, local_root)
    if rel_dir == ".":
        rel_dir = ""

    # Skip Obsidian internals and scripts -- don't sync them at all
    if ".obsidian" in rel_dir:
        dirnames[:] = []
        continue
    if rel_dir == "scripts" or rel_dir.startswith("scripts/"):
        dirnames[:] = []
        continue

    parent = campaign
    if rel_dir:
        parts = rel_dir.split(os.sep)
        current_path = ""
        for part in parts:
            current_path = f"{current_path}/{part}" if current_path else part
            parent = get_or_create_dir(parent, part, current_path)
            if parent is None:
                break
        if parent is None:
            errors += len(dirnames) + len(filenames)
            continue

    for filename in filenames:
        if ".obsidian" in filename:
            continue

        local_path = os.path.join(dirpath, filename)
        rel_path = os.path.join(rel_dir, filename) if rel_dir else filename

        with open(local_path, "rb") as f:
            local_data = f.read()

        if rel_path in icloud_files:
            existing = icloud_files[rel_path]
            if existing.size == len(local_data):
                skipped_unchanged += 1
                continue
            else:
                skipped_conflict += 1
                print(f"  [SKIP ] {rel_path} — differs from iCloud (edited elsewhere?)")
                continue

        try:
            file_obj = NamedBytesIO(local_data, filename)
            parent.upload(file_obj)
            uploaded += 1
            print(f"  [NEW  ] {rel_path} ({len(local_data)} bytes)")
        except Exception as e:
            print(f"  [ERR  ] {rel_path} — {e}")
            errors += 1

# Phase 2: Download iCloud files that don't exist locally.
print("\nPhase 2: Downloading new iCloud files to local...")
downloaded = 0
download_errors = 0

for rel_path, node in icloud_files.items():
    # Skip Obsidian internal files
    if "/.obsidian/" in rel_path or rel_path.startswith(".obsidian/"):
        continue

    local_path = os.path.join(local_root, rel_path)
    if os.path.exists(local_path):
        continue

    local_dir = os.path.dirname(local_path)
    os.makedirs(local_dir, exist_ok=True)

    try:
        with open(local_path, "wb") as f:
            f.write(node.open().read())
        downloaded += 1
        print(f"  [DOWN ] {rel_path} ({node.size} bytes)")
    except Exception as e:
        print(f"  [DERR ] {rel_path} — {e}")
        download_errors += 1

print(f"\n=== Sync Complete ===")
print(f"Uploaded:   {uploaded}")
print(f"Downloaded: {downloaded}")
print(f"Skipped (same):     {skipped_unchanged}")
print(f"Skipped (conflict): {skipped_conflict}")
print(f"Errors:     {errors + download_errors}")
```

### Key API Notes

- pyicloud `DriveNode` objects do NOT have `.get_content()`. Use `.open()` to get a file-like object, then `.read()`.
- `.open()` may return bytes or a stream depending on pyicloud version — `.read()` handles both.
- Directory creation via `.mkdir()` may return a dict instead of a node. Always re-fetch the actual child node.
- New directories sometimes need a second sync pass to resolve; the retry logic above handles this.

### Credential Mapping

The user's `.env` may store credentials under different names:
- `APPLE_ID_PASSWORD` → map to `APPLE_PASSWORD`
- `APPLE_APP_PASSWORD` → may be the app-specific password for 2FA
- Never log the password. Never write it to disk.

### Conflict Resolution

When `[SKIP]` appears for a conflict, the user must manually resolve:
1. Open both versions (iCloud on their Mac/iPad, local on the server).
2. Copy the desired version to the server.
3. Re-run sync — if sizes now match, it will be skipped as "same" on both sides.

To force a specific direction, the user can temporarily rename the unwanted version, sync, then delete the duplicate.