From d0c469c3c05eef7a37f5c49ca695c3e418e28748 Mon Sep 17 00:00:00 2001 From: Yakov Till <37628546+Lyapsus@users.noreply.github.com> Date: Mon, 27 Apr 2026 17:55:31 +0200 Subject: [PATCH] iclouddrive: add read only iCloud Photos support and SRP authentication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add read-only iCloud Photos support to the existing iclouddrive backend via `service = photos` config option. Also includes auth improvements on top of #9209's SRP authentication. **Photos features:** - 3-level hierarchy: libraries (Personal + Shared Photo Library) → albums → photos/videos - server-side smart albums (All Photos, Videos, Favorites, Screenshots, Live, Bursts, Panoramas, Slo-mo, Time-lapse, Portrait, Long Exposure, Animated, Hidden, Recently Deleted) - User-created albums and nested album folders - Live Photo `.MOV` companions as first-class entries - Edited photo versions (`-edited` suffix) and RAW alternatives - Duplicate filename dedup for camera counter wrap collisions - Parallel cold listing for large albums - Delta sync via CloudKit `changes/zone` - warm listings near-instant from disk cache - Disk cache (libraries, albums, photos) with atomic writes for crash safety - `ChangeNotify` support for FUSE mounts via `changes/zone` polling - `ListR` support for `--fast-list` and recursive operations - `--metadata` support - width, height, added-time, favorite, hidden - Fresh download URLs per file - no stale URL failures on long copies - FUSE mount documentation with recommended flags **Auth improvements over #9209:** - SMS 2FA fallback for users without trusted Apple devices - Explicit push notification request - fixes iOS/macOS 26.4+ where 409 no longer auto-pushes - Thread safety for concurrent FUSE callers (mutexes on session and client state) - Session endpoint caching - skips ~5s `/validate` round-trip on warm start - `Disconnect` support - clears auth state + disk cache - PCS cookie support for Advanced Data Protection accounts, including trusted-device approval for PCS cookies Built on @coughlanio's Photos PoC (Closes #8734) and @mikegillan's SRP auth (#9209). Fixes #7982 Co-authored-by: Chris Coughlan --- backend/iclouddrive/api/client.go | 132 +- backend/iclouddrive/api/photos.go | 2341 ++++++++++++++++++++++ backend/iclouddrive/api/photos_test.go | 1285 ++++++++++++ backend/iclouddrive/api/session.go | 668 +++--- backend/iclouddrive/api/session_test.go | 40 + backend/iclouddrive/api/srp.go | 26 +- backend/iclouddrive/api/srp_test.go | 46 +- backend/iclouddrive/icloud.go | 495 +++++ backend/iclouddrive/iclouddrive.go | 171 +- backend/iclouddrive/icloudphotos.go | 892 +++++++++ backend/iclouddrive/icloudphotos_test.go | 516 +++++ docs/content/iclouddrive.md | 159 +- 12 files changed, 6293 insertions(+), 478 deletions(-) create mode 100644 backend/iclouddrive/api/photos.go create mode 100644 backend/iclouddrive/api/photos_test.go create mode 100644 backend/iclouddrive/api/session_test.go create mode 100644 backend/iclouddrive/icloud.go create mode 100644 backend/iclouddrive/icloudphotos.go create mode 100644 backend/iclouddrive/icloudphotos_test.go diff --git a/backend/iclouddrive/api/client.go b/backend/iclouddrive/api/client.go index 350a4fc33..1bd96c918 100644 --- a/backend/iclouddrive/api/client.go +++ b/backend/iclouddrive/api/client.go @@ -1,4 +1,4 @@ -// Package api provides functionality for interacting with the iCloud API. +// Package api provides functionality for interacting with the iCloud API package api import ( @@ -8,16 +8,19 @@ import ( "errors" "fmt" "net/http" + "os" + "path/filepath" "strings" + "sync" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/lib/rest" ) const ( baseEndpoint = "https://www.icloud.com" - homeEndpoint = "https://www.icloud.com" setupEndpoint = "https://setup.icloud.com/setup/ws/1" authEndpoint = "https://idmsa.apple.com/appleauth/auth" ) @@ -28,26 +31,21 @@ type sessionSave func(*Session) type Client struct { appleID string password string + remoteName string // rclone remote name, used for cache namespacing srv *rest.Client Session *Session sessionSaveCallback sessionSave drive *DriveService + mu sync.Mutex // protects drive and Authenticate } -// New creates a new Client instance with the provided Apple ID, password, trust token, cookies, and session save callback. -// -// Parameters: -// - appleID: the Apple ID of the user. -// - password: the password of the user. -// - trustToken: the trust token for the session. -// - clientID: the client id for the session. -// - cookies: the cookies for the session. -// - sessionSaveCallback: the callback function to save the session. -func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave) (*Client, error) { +// New creates a new iCloud API client and initializes its HTTP session +func New(appleID, password, trustToken string, clientID string, cookies []*http.Cookie, sessionSaveCallback sessionSave, remoteName string) (*Client, error) { icloud := &Client{ - appleID: appleID, + appleID: strings.ToLower(appleID), // Apple SRP requires lowercase in client-side proof password: password, + remoteName: filepath.Base(remoteName), srv: rest.NewClient(fshttp.NewClient(context.Background())), Session: NewSession(), sessionSaveCallback: sessionSaveCallback, @@ -59,10 +57,12 @@ func New(appleID, password, trustToken string, clientID string, cookies []*http. return icloud, nil } -// DriveService returns the DriveService instance associated with the Client. +// DriveService returns the DriveService instance, creating it on first call func (c *Client) DriveService() (*DriveService, error) { - var err error + c.mu.Lock() + defer c.mu.Unlock() if c.drive == nil { + var err error c.drive, err = NewDriveService(c) if err != nil { return nil, err @@ -71,11 +71,7 @@ func (c *Client) DriveService() (*DriveService, error) { return c.drive, nil } -// Request makes a request and retries it if the session is invalid. -// -// This function is the main entry point for making requests to the iCloud -// API. If the initial request returns a 401 (Unauthorized), it will try to -// reauthenticate and retry the request. +// Request makes a request to the iCloud API, re-authenticating on 401/421 func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) { resp, err = c.Session.Request(ctx, opts, request, response) if err != nil && resp != nil { @@ -89,55 +85,93 @@ func (c *Client) Request(ctx context.Context, opts rest.Opts, request any, respo if c.Session.Requires2FA() { return nil, errors.New("trust token expired, please reauth") } - return c.RequestNoReAuth(ctx, opts, request, response) + return c.Session.Request(ctx, opts, request, response) } } return resp, err } -// RequestNoReAuth makes a request without re-authenticating. -// -// This function is useful when you have a session that is already -// authenticated, but you need to make a request without triggering -// a re-authentication. -func (c *Client) RequestNoReAuth(ctx context.Context, opts rest.Opts, request any, response any) (resp *http.Response, err error) { - // Make the request without re-authenticating - resp, err = c.Session.Request(ctx, opts, request, response) - return resp, err -} - -// Authenticate authenticates the client with the iCloud API. +// Authenticate authenticates the client, reusing existing session if valid func (c *Client) Authenticate(ctx context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + // Skip /validate round-trip when saved session has cookies + service endpoints + // Native client behavior: use cached session, reauth lazily on 401/421 + if c.Session.Cookies != nil && len(c.Session.AccountInfo.Webservices) > 0 { + fs.Debugf(nil, "iclouddrive: reusing saved session") + return nil + } + // Try loading cached service endpoints to avoid /validate round-trip (~5s) + if c.Session.Cookies != nil && c.loadCachedWebservices() { + fs.Debugf(nil, "iclouddrive: reusing session with cached endpoints") + return nil + } if c.Session.Cookies != nil { if err := c.Session.ValidateSession(ctx); err == nil { - fs.Debugf("icloud", "Valid session, no need to reauth") + fs.Debugf(nil, "iclouddrive: valid session, no need to reauth") + c.saveCachedWebservices() return nil } c.Session.Cookies = nil } - fs.Debugf("icloud", "Authenticating as %s\n", c.appleID) + fs.Debugf(nil, "iclouddrive: authenticating") err := c.Session.SignIn(ctx, c.appleID, c.password) if err != nil { return err } - // If 2FA is required, don't try AuthWithToken yet — the caller - // must complete 2FA first, then call AuthWithToken. + // If 2FA is required, skip AuthWithToken - caller must complete 2FA first if c.Session.Requires2FA() { return nil } err = c.Session.AuthWithToken(ctx) - if err == nil && c.sessionSaveCallback != nil { - c.sessionSaveCallback(c.Session) + if err == nil { + c.saveCachedWebservices() + if c.sessionSaveCallback != nil { + c.sessionSaveCallback(c.Session) + } } return err } -// SignIn signs in the client using the provided context and credentials. -func (c *Client) SignIn(ctx context.Context) error { - return c.Session.SignIn(ctx, c.appleID, c.password) +// loadCachedWebservices loads service endpoints from disk cache +func (c *Client) loadCachedWebservices() bool { + data, err := os.ReadFile(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName, "webservices.json")) + if err != nil { + return false + } + var ws map[string]*webService + if err := json.Unmarshal(data, &ws); err != nil { + return false + } + if len(ws) == 0 { + return false + } + c.Session.AccountInfo.Webservices = ws + return true +} + +// saveCachedWebservices persists service endpoints to disk +func (c *Client) saveCachedWebservices() { + if len(c.Session.AccountInfo.Webservices) == 0 { + return + } + saveJSONCache(filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName), "webservices.json", c.Session.AccountInfo.Webservices) +} + +// CacheDir returns the disk cache directory for this remote +func (c *Client) CacheDir() string { + return filepath.Join(config.GetCacheDir(), cacheSubdir, c.remoteName) +} + +// ClearCacheDir removes all disk cache files for a remote +func ClearCacheDir(remoteName string) { + dir := filepath.Join(config.GetCacheDir(), cacheSubdir, filepath.Base(remoteName)) + if err := os.RemoveAll(dir); err != nil { + fs.Debugf(nil, "iclouddrive: failed to clear cache: %v", err) + } } // IntoReader marshals the provided values into a JSON encoded reader @@ -155,19 +189,19 @@ type RequestError struct { Text string } -// Error satisfy the error interface. +// Error satisfies the error interface func (e *RequestError) Error() string { return fmt.Sprintf("%s: %s", e.Text, e.Status) } -func newRequestError(Status string, Text string) *RequestError { +func newRequestError(status string, text string) *RequestError { return &RequestError{ - Status: strings.ToLower(Status), - Text: Text, + Status: strings.ToLower(status), + Text: text, } } -// newErr orf makes a new error from sprintf parameters. -func newRequestErrorf(Status string, Text string, Parameters ...any) *RequestError { - return newRequestError(strings.ToLower(Status), fmt.Sprintf(Text, Parameters...)) +// newRequestErrorf makes a new error from sprintf parameters +func newRequestErrorf(status string, text string, params ...any) *RequestError { + return newRequestError(status, fmt.Sprintf(text, params...)) } diff --git a/backend/iclouddrive/api/photos.go b/backend/iclouddrive/api/photos.go new file mode 100644 index 000000000..06cdc226e --- /dev/null +++ b/backend/iclouddrive/api/photos.go @@ -0,0 +1,2341 @@ +package api + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "sync" + "sync/atomic" + + "github.com/rclone/rclone/fs" + + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" + + "golang.org/x/text/unicode/norm" +) + +const ( + // CloudKit returns CPLMaster+CPLAsset pairs, so 200 records = 100 photos + // Server caps at 200 regardless of requested value (tested: 500-5000 all return 200) + photosQueryLimit = 200 + rootFolderRecord = "----Root-Folder----" + projectRootFolderRecord = "----Project-Root-Folder----" + indexingStateReady = "FINISHED" + + // cacheSubdir is the subdirectory under rclone's cache dir for all iCloud Photos state + cacheSubdir = "iclouddrive-photos" + + // Album type constants from CloudKit CPLAlbum records + albumTypeFolder = 3 + + // CPLAsset.assetSubtype values + subtypePanorama = 1 + subtypeSloMo = 100 + subtypeTimeLapse = 101 + + // CPLAsset.assetSubtypeV2 values + subtypeV2Live = 2 + subtypeV2Screenshot = 3 + + // CPLAsset.adjustmentRenderType bitmask values + adjustPortrait = 2 + adjustLongExposure = 4 + + // CloudKit record type names + recordTypeAlbum = "CPLAlbumByPositionLive" + recordTypeCountIndex = "HyperionIndexCountLookup" + + // CloudKit endpoint area names + areaPrivate = "private" + areaShared = "shared" + + // Slo-mo adjustment type (metadata-only edit, no separate rendered resource) + adjustSloMo = "com.apple.video.slomo" +) + +// utiExtensions maps common Apple UTI descriptors to file extensions +// Used as fallback when filenameEnc is missing from a CPLMaster record +var utiExtensions = map[string]string{ + "public.jpeg": ".jpg", + "public.png": ".png", + "public.heic": ".heic", + "public.heif": ".heif", + "public.tiff": ".tiff", + "public.mpeg-4": ".mp4", + "com.apple.quicktime-movie": ".mov", + "com.compuserve.gif": ".gif", + "com.adobe.raw-image": ".dng", + "com.canon.cr2-raw-image": ".cr2", + "com.canon.cr3-raw-image": ".cr3", + "com.nikon.raw-image": ".nef", + "com.sony.arw-raw-image": ".arw", + "public.avif": ".avif", + "org.webmproject.webp": ".webp", + "public.mpeg-2-video": ".m2v", + "com.apple.m4v-video": ".m4v", + "public.avi": ".avi", + "public.mp3": ".mp3", + "com.apple.m4a-audio": ".m4a", + "public.image": ".heic", + "com.fuji.raw-image": ".raf", + "com.panasonic.rw2-raw-image": ".rw2", + "com.olympus.raw-image": ".orf", + "com.pentax.raw-image": ".pef", + "com.nikon.nrw-raw-image": ".nrw", + "com.canon.crw-raw-image": ".crw", +} + +// extFromUTI returns the file extension for an Apple UTI string field, +// falling back to the provided default if the field is nil or unknown +func extFromUTI(field *ckStringField, fallback string) string { + if field != nil { + if mapped, ok := utiExtensions[field.Value]; ok { + return mapped + } + } + return fallback +} + +// buildPhotoCache creates a filename-keyed map from a photo slice +func buildPhotoCache(photos []*Photo) map[string]*Photo { + m := make(map[string]*Photo, len(photos)) + for _, p := range photos { + if p.Filename != "" { + m[p.Filename] = p + } + } + return m +} + +// saveJSONCache marshals v to JSON and writes it atomically to dir/filename +func saveJSONCache(dir, filename string, v any) { + if err := os.MkdirAll(dir, 0700); err != nil { + fs.Debugf(nil, "iclouddrive: failed to create cache dir: %v", err) + return + } + data, err := json.Marshal(v) + if err != nil { + fs.Debugf(nil, "iclouddrive: failed to marshal cache %s: %v", filename, err) + return + } + if err := atomicWriteFile(filepath.Join(dir, filename), data); err != nil { + fs.Debugf(nil, "iclouddrive: failed to write cache %s: %v", filename, err) + } +} + +// atomicWriteFile writes data to target via atomic temp+rename +func atomicWriteFile(target string, data []byte) error { + tmp := target + ".tmp" + if err := os.WriteFile(tmp, data, 0600); err != nil { + return err + } + if err := os.Rename(tmp, target); err != nil { + _ = os.Remove(tmp) + return err + } + return nil +} + +// ShouldRetryFunc classifies whether an HTTP response/error is retryable +type ShouldRetryFunc func(ctx context.Context, resp *http.Response, err error) (bool, error) + +// PhotosService manages iCloud Photos API interactions +type PhotosService struct { + client *Client + endpoint string + pacer *fs.Pacer + shouldRetry ShouldRetryFunc + mu sync.Mutex + libraries map[string]*Library +} + +type libraryDiscovery struct { + libraries map[string]*Library + refreshedAreas map[string]bool +} + +// FlushCaches clears all cached libraries, albums, and photos +func (ps *PhotosService) FlushCaches() { + ps.mu.Lock() + defer ps.mu.Unlock() + for _, lib := range ps.libraries { + lib.deltaMu.Lock() + lib.mu.Lock() + lib.cacheValid.Store(false) + lib.pendingDelta = nil + lib.clearDiskCache() + for _, album := range lib.albums { + album.mu.Lock() + album.photoCache = nil + album.mu.Unlock() + } + lib.albums = make(map[string]*Album) + lib.mu.Unlock() + lib.deltaMu.Unlock() + } + ps.libraries = make(map[string]*Library) + // Also remove the libraries cache file + _ = os.Remove(filepath.Join(ps.client.CacheDir(), "libraries.json")) +} + +// deltaPayload holds a buffered changes/zone response waiting to be applied +// after albums are populated +type deltaPayload struct { + records []json.RawMessage + syncToken string + moreComing bool +} + +// deltaContainsAlbumChanges checks if any record in the delta is a CPLAlbum +// (album create/rename/delete) requiring eager album cache invalidation +func deltaContainsAlbumChanges(records []json.RawMessage) bool { + for _, raw := range records { + var header struct { + RecordType string `json:"recordType"` + } + if json.Unmarshal(raw, &header) == nil && header.RecordType == "CPLAlbum" { + return true + } + } + return false +} + +// Library represents a photo library in a specific zone +type Library struct { + service *PhotosService + zoneID string + area string // "private" or "shared" - determines API endpoint path + ownerRecordName string // zone owner's _UUID for full zoneID in requests + zoneType string // "REGULAR_CUSTOM_ZONE" for full zoneID in requests + mu sync.Mutex // protects albums map + albums map[string]*Album + deltaMu sync.Mutex // serializes delta checks+apply; lock order: ps.mu before deltaMu + cacheValid atomic.Bool // true = album/photo disk caches are loadable + pendingDelta *deltaPayload // buffered delta waiting for albums to be populated, protected by deltaMu + notifyToken string // separate changes/zone token for ChangeNotify polling (memory-only) +} + +// zoneIDMap returns the full zoneID object for CloudKit API requests +func (lib *Library) zoneIDMap() map[string]any { + m := map[string]any{"zoneName": lib.zoneID} + if lib.ownerRecordName != "" { + m["ownerRecordName"] = lib.ownerRecordName + } + if lib.zoneType != "" { + m["zoneType"] = lib.zoneType + } + return m +} + +// invalidateAlbumCache clears in-memory albums and removes disk cache +func (lib *Library) invalidateAlbumCache() { + lib.mu.Lock() + lib.albums = make(map[string]*Album) + lib.mu.Unlock() + _ = os.Remove(filepath.Join(lib.zoneCacheDir(), "albums.json")) +} + +// bufferDelta stores a pending delta for later application, eagerly +// invalidating album cache if the delta contains CPLAlbum records +// Must be called under deltaMu +func (lib *Library) bufferDelta(records []json.RawMessage, syncToken string, moreComing bool) { + lib.pendingDelta = &deltaPayload{records: records, syncToken: syncToken, moreComing: moreComing} + lib.cacheValid.Store(true) + fs.Debugf(nil, "iclouddrive photos: zone %s has pending changes, buffered for later application", lib.zoneID) + if deltaContainsAlbumChanges(records) { + lib.invalidateAlbumCache() + fs.Debugf(nil, "iclouddrive photos: zone %s delta contains album changes, invalidated album cache", lib.zoneID) + } +} + +// request makes an API call routed to this library's area (private or shared) +func (lib *Library) request(ctx context.Context, endpoint string, data, response any) error { + return lib.service.requestForArea(ctx, lib.area, endpoint, data, response) +} + +func (lib *Library) isSharedLibrary() bool { + return strings.HasPrefix(lib.zoneID, "SharedSync") +} + +func isSharedAlbumIndexError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "HTTP error 400") && + strings.Contains(msg, "BAD_REQUEST") && + strings.Contains(msg, "Index has invalid data") +} + +func isZoneNotFoundError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "HTTP error 404") && + strings.Contains(msg, "ZONE_NOT_FOUND") && + strings.Contains(msg, "Zone does not exist") +} + +func (lib *Library) probeZoneExists(ctx context.Context) (bool, error) { + query := map[string]any{ + "query": map[string]any{ + "recordType": "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted", + "filterBy": []map[string]any{ + { + "fieldName": "startRank", + "comparator": "EQUALS", + "fieldValue": map[string]any{"type": "INT64", "value": 0}, + }, + { + "fieldName": "direction", + "comparator": "EQUALS", + "fieldValue": map[string]any{"type": "STRING", "value": "ASCENDING"}, + }, + }, + }, + "resultsLimit": 1, + "desiredKeys": []string{"masterRef"}, + "zoneID": lib.zoneIDMap(), + } + var response struct { + Records []json.RawMessage `json:"records"` + } + if err := lib.request(ctx, "records/query", query, &response); err != nil { + if isZoneNotFoundError(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// newUserAlbum creates a standard user album with CPLContainerRelation query config +func (lib *Library) newUserAlbum(name, recordName string) *Album { + return &Album{ + Name: name, + ObjectType: fmt.Sprintf("CPLContainerRelationNotDeletedByAssetDate:%s", recordName), + ListType: "CPLContainerRelationLiveByAssetDate", + Direction: "ASCENDING", + RecordName: recordName, + lib: lib, + Filters: []Filter{{ + FieldName: "parentId", + Comparator: "EQUALS", + FieldValue: map[string]string{"type": "STRING", "value": recordName}, + }}, + } +} + +// Album represents a photo album with its metadata and query configuration +type Album struct { + Name string `json:"name"` + ObjectType string `json:"objectType"` + ListType string `json:"listType"` + Direction string `json:"direction"` + Filters []Filter `json:"filters,omitempty"` + RecordName string `json:"recordName,omitempty"` + IsFolder bool `json:"isFolder,omitempty"` + Children map[string]*Album `json:"children,omitempty"` + lib *Library `json:"-"` + mu sync.Mutex `json:"-"` + photoCache map[string]*Photo `json:"-"` +} + +// Photo represents a photo or video with its metadata +type Photo struct { + ID string + Filename string + Size int64 + AssetDate int64 // Unix timestamp in milliseconds + AddedDate int64 + Width int + Height int + IsFavorite bool + IsHidden bool + SmartAlbums []string // smart album names this photo belongs to (for delta sync routing) + ResourceKey string // CloudKit field name for download (default: resOriginalRes) +} + +// companion creates a derivative Photo entry sharing metadata with the parent +// Used for Live Photo MOV, edited versions, and RAW alternatives +func (p *Photo) companion(id, filename, resourceKey string, size int64) *Photo { + return &Photo{ + ID: id, + Filename: filename, + Size: size, + AssetDate: p.AssetDate, + AddedDate: p.AddedDate, + IsFavorite: p.IsFavorite, + IsHidden: p.IsHidden, + ResourceKey: resourceKey, + SmartAlbums: p.SmartAlbums, + } +} + +// Filter represents a CloudKit query filter +type Filter struct { + FieldName string `json:"fieldName"` + Comparator string `json:"comparator"` + FieldValue any `json:"fieldValue"` +} + +// smartAlbumFilter defines a filter-based smart album generated from a short table +// Each entry maps: display name -> ObjectType suffix + filter tag +// All share ListType "CPLAssetAndMasterInSmartAlbumByAssetDate", direction ASCENDING, +// and a single smartAlbum EQUALS filter +type smartAlbumFilter struct { + suffix string // appended to "CPLAssetInSmartAlbumByAssetDate:" + tag string // smartAlbum filter value (e.g. "VIDEO", "FAVORITE") +} + +// smartAlbumFilters is the data table for the 10 filter-based smart albums +var smartAlbumFilters = map[string]smartAlbumFilter{ + "Time-lapse": {suffix: "Timelapse", tag: "TIMELAPSE"}, + "Videos": {suffix: "Video", tag: "VIDEO"}, + "Slo-mo": {suffix: "Slomo", tag: "SLOMO"}, + "Favorites": {suffix: "Favorite", tag: "FAVORITE"}, + "Panoramas": {suffix: "Panorama", tag: "PANORAMA"}, + "Screenshots": {suffix: "Screenshot", tag: "SCREENSHOT"}, + "Live": {suffix: "Live", tag: "LIVE"}, + "Portrait": {suffix: "Depth", tag: "DEPTH"}, + "Long Exposure": {suffix: "Exposure", tag: "EXPOSURE"}, + "Animated": {suffix: "Animated", tag: "ANIMATED"}, + // SELFIE filter exists server-side but the index is never populated via web API - + // selfie classification is on-device only (iOS reads LensModel EXIF for "front camera") + // Apple's own icloud.com doesn't show it either - omitted to avoid an always-empty album +} + +// SmartAlbums defines the built-in smart album types available in iCloud Photos +// 10 filter-based albums are generated from smartAlbumFilters; 4 special albums +// (All Photos, Bursts, Hidden, Recently Deleted) use unique recordTypes +var SmartAlbums = buildSmartAlbums() + +func buildSmartAlbums() map[string]*Album { + albums := map[string]*Album{ + "All Photos": { + Name: "All Photos", + ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted", + ListType: "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted", + Direction: "ASCENDING", + }, + "Bursts": { + Name: "Bursts", + ObjectType: "CPLAssetBurstStackAssetByAssetDate", + ListType: "CPLBurstStackAssetAndMasterByAssetDate", + Direction: "ASCENDING", + }, + "Hidden": { + Name: "Hidden", + ObjectType: "CPLAssetHiddenByAssetDate", + ListType: "CPLAssetAndMasterHiddenByAssetDate", + Direction: "ASCENDING", + }, + "Recently Deleted": { + Name: "Recently Deleted", + ObjectType: "CPLAssetDeletedByExpungedDate", + ListType: "CPLAssetAndMasterDeletedByExpungedDate", + Direction: "DESCENDING", + }, + } + for name, f := range smartAlbumFilters { + albums[name] = &Album{ + Name: name, + ObjectType: "CPLAssetInSmartAlbumByAssetDate:" + f.suffix, + ListType: "CPLAssetAndMasterInSmartAlbumByAssetDate", + Direction: "ASCENDING", + Filters: []Filter{{ + FieldName: "smartAlbum", + Comparator: "EQUALS", + FieldValue: map[string]string{"type": "STRING", "value": f.tag}, + }}, + } + } + return albums +} + +type errorRoundTripper struct{} + +func (errorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, errors.New("test photos service has no network transport") +} + +// NewTestPhotosService creates a PhotosService with pre-populated libraries for testing +func NewTestPhotosService(libs map[string]map[string]*Album) *PhotosService { + httpClient := &http.Client{Transport: errorRoundTripper{}} + session := &Session{srv: rest.NewClient(httpClient)} + ps := &PhotosService{ + client: &Client{ + remoteName: "_test", + Session: session, + }, + endpoint: "http://test.invalid/database/1/com.apple.photos.cloud/production", + pacer: fs.NewPacer(context.Background(), pacer.NewDefault()), + shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err }, + libraries: make(map[string]*Library), + } + for zoneName, albums := range libs { + lib := &Library{ + service: ps, + zoneID: zoneName, + area: areaPrivate, + albums: make(map[string]*Album), + } + lib.cacheValid.Store(true) + for name, album := range albums { + lib.restoreAlbumLinks(album) + lib.albums[name] = album + } + ps.libraries[zoneName] = lib + } + return ps +} + +// SetTestPhotoCache populates an album's photo cache for testing +func (album *Album) SetTestPhotoCache(cache map[string]*Photo) { + album.mu.Lock() + album.photoCache = cache + album.mu.Unlock() +} + +// NewPhotosService creates a new PhotosService instance +func NewPhotosService(ctx context.Context, client *Client, pacer *fs.Pacer, shouldRetry ShouldRetryFunc) (*PhotosService, error) { + service, exists := client.Session.AccountInfo.Webservices["ckdatabasews"] + if !exists || service.Status != "active" { + return nil, fmt.Errorf("ckdatabasews service not available") + } + endpoint := fmt.Sprintf("%s/database/1/com.apple.photos.cloud/production", service.URL) + + ps := &PhotosService{ + client: client, + endpoint: endpoint, + pacer: pacer, + shouldRetry: shouldRetry, + libraries: make(map[string]*Library), + } + + ps.checkIndexingState(ctx, "PrimarySync") + + return ps, nil +} + +// GetLibraries returns all available photo libraries +func (ps *PhotosService) GetLibraries(ctx context.Context) (map[string]*Library, error) { + ps.mu.Lock() + defer ps.mu.Unlock() + + if len(ps.libraries) > 0 { + if discovered, err := ps.discoverLibraries(ctx); err == nil { + ps.libraries = mergeDiscoveredLibraries(ctx, ps.libraries, discovered) + ps.saveCachedLibraries() + fs.Debugf(nil, "iclouddrive photos: refreshed %d in-memory libraries from API", len(ps.libraries)) + } else { + fs.Debugf(nil, "iclouddrive photos: in-memory library rediscovery failed, using cached state: %v", err) + } + return ps.libraries, nil + } + + // Try loading cached zone names from disk + if cached := ps.loadCachedLibraries(); cached != nil { + ps.batchCheckForChanges(ctx, cached) + ps.libraries = cached + if discovered, err := ps.discoverLibraries(ctx); err == nil { + ps.libraries = mergeDiscoveredLibraries(ctx, cached, discovered) + ps.saveCachedLibraries() + fs.Debugf(nil, "iclouddrive photos: refreshed %d libraries from API", len(ps.libraries)) + } else { + fs.Debugf(nil, "iclouddrive photos: library rediscovery failed, using cached zones: %v", err) + fs.Debugf(nil, "iclouddrive photos: %d libraries from cache", len(cached)) + } + return ps.libraries, nil + } + + discovered, err := ps.discoverLibraries(ctx) + if err != nil { + return nil, err + } + ps.libraries = discovered.libraries + ps.saveCachedLibraries() + return ps.libraries, nil +} + +func (ps *PhotosService) discoverLibraries(ctx context.Context) (*libraryDiscovery, error) { + result := &libraryDiscovery{ + libraries: make(map[string]*Library), + refreshedAreas: make(map[string]bool), + } + + // Discover zones from API - probe both private and shared databases + // Private zones: owned by the current user (PrimarySync + owned SharedSync) + // Shared zones: owned by another user (non-owner SharedSync) + type zoneResponse struct { + Zones []struct { + ZoneID struct { + ZoneName string `json:"zoneName"` + OwnerRecordName string `json:"ownerRecordName"` + ZoneType string `json:"zoneType"` + } `json:"zoneID"` + Deleted bool `json:"deleted,omitempty"` + } `json:"zones"` + } + + for _, area := range []string{areaPrivate, areaShared} { + var response zoneResponse + if err := ps.requestForArea(ctx, area, "changes/database", map[string]any{}, &response); err != nil { + if area == areaShared { + // Shared database may not exist for all accounts + fs.Debugf(nil, "iclouddrive photos: shared zone discovery failed (expected if no shared library): %v", err) + continue + } + return nil, fmt.Errorf("failed to discover zones: %w", err) + } + result.refreshedAreas[area] = true + for _, zone := range response.Zones { + if zone.Deleted { + continue + } + name := zone.ZoneID.ZoneName + // SharedSync-* found in private takes precedence over shared + if _, exists := result.libraries[name]; exists { + continue + } + result.libraries[name] = &Library{ + service: ps, + zoneID: name, + area: area, + ownerRecordName: zone.ZoneID.OwnerRecordName, + zoneType: zone.ZoneID.ZoneType, + albums: make(map[string]*Album), + } + } + } + return result, nil +} + +func mergeDiscoveredLibraries(ctx context.Context, existing map[string]*Library, discovered *libraryDiscovery) map[string]*Library { + merged := make(map[string]*Library, len(existing)+len(discovered.libraries)) + for name, cached := range existing { + if !discovered.refreshedAreas[cached.area] { + if cached.area == areaShared { + exists, err := cached.probeZoneExists(ctx) + if err != nil { + fs.Debugf(nil, "iclouddrive photos: shared zone probe failed for %q, keeping cached zone: %v", cached.zoneID, err) + } else if !exists { + fs.Debugf(nil, "iclouddrive photos: dropping cached shared zone %q after authoritative ZONE_NOT_FOUND probe", cached.zoneID) + continue + } + } + merged[name] = cached + } + } + for name, fresh := range discovered.libraries { + if cached, ok := existing[name]; ok { + cached.service = fresh.service + cached.area = fresh.area + cached.ownerRecordName = fresh.ownerRecordName + cached.zoneType = fresh.zoneType + merged[name] = cached + continue + } + merged[name] = fresh + } + return merged +} + +// albumRecord represents a CPLAlbum record from CloudKit +type albumRecord struct { + RecordName string `json:"recordName"` + Fields struct { + AlbumNameEnc *ckStringField `json:"albumNameEnc,omitempty"` + AlbumType *ckIntField `json:"albumType,omitempty"` + ParentID *ckStringField `json:"parentId,omitempty"` + IsDeleted *ckBoolField `json:"isDeleted,omitempty"` + } `json:"fields"` +} + +// albumQueryResponse wraps a paginated list of album records +type albumQueryResponse struct { + Records []albumRecord `json:"records"` + ContinuationMarker string `json:"continuationMarker"` +} + +// cachedLibraryEntry stores zone metadata for disk cache persistence +type cachedLibraryEntry struct { + ZoneName string `json:"zoneName"` + Area string `json:"area,omitempty"` + OwnerRecordName string `json:"ownerRecordName,omitempty"` + ZoneType string `json:"zoneType,omitempty"` +} + +// loadCachedLibraries loads zone metadata from disk cache +func (ps *PhotosService) loadCachedLibraries() map[string]*Library { + cacheFile := filepath.Join(ps.client.CacheDir(), "libraries.json") + data, err := os.ReadFile(cacheFile) + if err != nil { + return nil + } + + var entries []cachedLibraryEntry + if err := json.Unmarshal(data, &entries); err != nil { + return nil + } + + libs := make(map[string]*Library, len(entries)) + for _, entry := range entries { + area := entry.Area + if area == "" { + area = areaPrivate + } + libs[entry.ZoneName] = &Library{ + service: ps, + zoneID: entry.ZoneName, + area: area, + ownerRecordName: entry.OwnerRecordName, + zoneType: entry.ZoneType, + albums: make(map[string]*Album), + } + } + return libs +} + +// saveCachedLibraries persists zone metadata to disk via atomic rename +func (ps *PhotosService) saveCachedLibraries() { + var entries []cachedLibraryEntry + for _, lib := range ps.libraries { + entries = append(entries, cachedLibraryEntry{ + ZoneName: lib.zoneID, + Area: lib.area, + OwnerRecordName: lib.ownerRecordName, + ZoneType: lib.zoneType, + }) + } + saveJSONCache(ps.client.CacheDir(), "libraries.json", entries) +} + +// GetAlbums returns all albums for this library +func (lib *Library) GetAlbums(ctx context.Context) (map[string]*Album, error) { + lib.mu.Lock() + defer lib.mu.Unlock() + + if len(lib.albums) > 0 { + return lib.albums, nil + } + + // Try loading cached albums if zone is unchanged + if lib.cacheValid.Load() { + if cached := lib.loadCachedAlbums(); cached != nil { + lib.albums = cached + fs.Debugf(nil, "iclouddrive photos: %d albums from cache for zone %s", len(cached), lib.zoneID) + return lib.albums, nil + } + } + + // Build albums into a local map first so that a transient user album + // query failure leaves lib.albums empty (retried on next call) rather + // than permanently caching the smart-album-only subset + albums := make(map[string]*Album, len(SmartAlbums)) + + // Add smart albums + for name, template := range SmartAlbums { + albums[name] = &Album{ + Name: template.Name, + ObjectType: template.ObjectType, + ListType: template.ListType, + Direction: template.Direction, + Filters: append([]Filter{}, template.Filters...), + RecordName: template.RecordName, + lib: lib, + } + } + + // Add user albums and folders (paginated - CloudKit caps at 200 per page) + var allRecords []albumRecord + var continuationMarker string + + for { + query := map[string]any{ + "query": map[string]any{"recordType": recordTypeAlbum}, + "zoneID": lib.zoneIDMap(), + "desiredKeys": []string{"albumNameEnc", "albumType", "parentId", "isDeleted"}, + } + if continuationMarker != "" { + query["continuationMarker"] = continuationMarker + } + + var response albumQueryResponse + + if err := lib.request(ctx, "records/query", query, &response); err != nil { + // SharedSync libraries return BAD_REQUEST / "Index has invalid data" + // on CPLAlbumByPositionLive in live probes, so fall back to smart + // albums there while surfacing PrimarySync failures directly + fs.Debugf(nil, "iclouddrive photos: user album query failed for zone %q: %v", lib.zoneID, err) + if lib.isSharedLibrary() && isSharedAlbumIndexError(err) { + return albums, nil + } + return nil, fmt.Errorf("query user albums for zone %q: %w", lib.zoneID, err) + } + + allRecords = append(allRecords, response.Records...) + + if response.ContinuationMarker == "" { + break + } + continuationMarker = response.ContinuationMarker + } + + for _, record := range allRecords { + if record.Fields.AlbumNameEnc == nil || + record.RecordName == rootFolderRecord || + record.RecordName == projectRootFolderRecord || + (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) { + continue + } + + nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value) + if err != nil { + fs.Debugf(nil, "iclouddrive photos: skipping album %q: base64 decode: %v", record.RecordName, err) + continue + } + + isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder + albumName := norm.NFC.String(string(nameBytes)) + + // User album with same name as a smart album - smart album has special + // server-side query semantics that can't be replicated by the user album, + // so we keep the smart album and skip the user album with a warning + if _, isSmart := SmartAlbums[albumName]; isSmart && !isFolder { + fs.Logf(nil, "iclouddrive photos: user album %q shadows smart album, using smart album", albumName) + continue + } + + if isFolder { + folder := &Album{ + Name: albumName, + RecordName: record.RecordName, + lib: lib, + IsFolder: true, + Children: make(map[string]*Album), + } + if err := lib.fetchFolderChildren(ctx, folder); err != nil { + return nil, fmt.Errorf("fetch children of folder %q: %w", albumName, err) + } + albums[albumName] = folder + } else { + albums[albumName] = lib.newUserAlbum(albumName, record.RecordName) + } + } + + lib.albums = albums + lib.saveCachedAlbums() + return lib.albums, nil +} + +// loadCachedAlbums loads album metadata from disk cache +func (lib *Library) loadCachedAlbums() map[string]*Album { + cacheFile := filepath.Join(lib.zoneCacheDir(), "albums.json") + data, err := os.ReadFile(cacheFile) + if err != nil { + return nil + } + var albums map[string]*Album + if err := json.Unmarshal(data, &albums); err != nil { + return nil + } + // Restore runtime-only fields after deserialization + for _, a := range albums { + lib.restoreAlbumLinks(a) + } + return albums +} + +// restoreAlbumLinks sets runtime-only fields (lib pointer) on an album +// and its children after deserialization from disk cache +func (lib *Library) restoreAlbumLinks(a *Album) { + a.lib = lib + for _, child := range a.Children { + lib.restoreAlbumLinks(child) + } +} + +func flattenAlbumTree(dst []*Album, albums map[string]*Album) []*Album { + for _, album := range albums { + dst = append(dst, album) + if album.IsFolder { + dst = flattenAlbumTree(dst, album.Children) + } + } + return dst +} + +// saveCachedAlbums persists album metadata to disk via atomic rename +func (lib *Library) saveCachedAlbums() { + saveJSONCache(lib.zoneCacheDir(), "albums.json", lib.albums) +} + +// fetchFolderChildren queries child albums inside a folder by parentId +func (lib *Library) fetchFolderChildren(ctx context.Context, folder *Album) error { + query := map[string]any{ + "query": map[string]any{ + "recordType": recordTypeAlbum, + "filterBy": []map[string]any{{ + "fieldName": "parentId", + "comparator": "EQUALS", + "fieldValue": map[string]string{"type": "STRING", "value": folder.RecordName}, + }}, + }, + "zoneID": lib.zoneIDMap(), + "desiredKeys": []string{"albumNameEnc", "albumType", "isDeleted"}, + } + + var continuationMarker string + for { + if continuationMarker != "" { + query["continuationMarker"] = continuationMarker + } + + var response albumQueryResponse + + if err := lib.request(ctx, "records/query", query, &response); err != nil { + return err + } + + for _, record := range response.Records { + if record.Fields.AlbumNameEnc == nil || + (record.Fields.IsDeleted != nil && record.Fields.IsDeleted.Value) { + continue + } + + nameBytes, err := base64.StdEncoding.DecodeString(record.Fields.AlbumNameEnc.Value) + if err != nil { + fs.Debugf(nil, "iclouddrive photos: skipping child album %q: base64 decode: %v", record.RecordName, err) + continue + } + + childName := norm.NFC.String(string(nameBytes)) + isFolder := record.Fields.AlbumType != nil && record.Fields.AlbumType.Value == albumTypeFolder + + if isFolder { + childFolder := &Album{ + Name: childName, + RecordName: record.RecordName, + lib: lib, + IsFolder: true, + Children: make(map[string]*Album), + } + if err := lib.fetchFolderChildren(ctx, childFolder); err != nil { + return err + } + folder.Children[childName] = childFolder + } else { + folder.Children[childName] = lib.newUserAlbum(childName, record.RecordName) + } + } + + if response.ContinuationMarker == "" { + break + } + continuationMarker = response.ContinuationMarker + } + + return nil +} + +// albumCacheKey returns a stable filename-safe key for an album's disk cache +func albumCacheKey(objectType string) string { + h := sha256.Sum256([]byte(objectType)) + return hex.EncodeToString(h[:8]) +} + +// zoneCacheDir returns the disk cache directory for this zone +// Path follows rclone convention: //// +func (lib *Library) zoneCacheDir() string { + return filepath.Join(lib.service.client.CacheDir(), lib.zoneID) +} + +// checkForChanges detects whether the zone has been modified since the last +// sync. If unchanged (0 records), sets cacheValid=true. If changed, buffers +// the first page as pendingDelta and still sets cacheValid=true (album disk +// cache is valid under the old token - delta hasn't been applied yet) +// The buffered delta is consumed later by applyPendingDelta when albums exist +func (lib *Library) checkForChanges(ctx context.Context) { + lib.deltaMu.Lock() + defer lib.deltaMu.Unlock() + + // Already have a buffered delta waiting to be applied + if lib.pendingDelta != nil { + return + } + + token := lib.readSyncToken() + if token == "" { + return + } + + var response changesZoneResponse + if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(token), &response); err != nil { + fs.Debugf(nil, "iclouddrive photos: delta check failed for zone %s: %v", lib.zoneID, err) + return + } + if len(response.Zones) == 0 { + return + } + + zone := response.Zones[0] + + if len(zone.Records) == 0 && !zone.MoreComing { + // No changes - advance token, all caches valid + fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID) + lib.saveSyncToken(zone.SyncToken) + lib.cacheValid.Store(true) + return + } + + // Buffer the delta for later application (after albums are populated) + // Album disk cache is still valid under the old token + lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing) +} + +// zoneEntry pairs a changes/zone request body with its library for batched zone operations +type zoneEntry struct { + zone map[string]any + lib *Library +} + +// flattenZoneEntries extracts the zone request bodies and builds a zoneID→Library lookup +func flattenZoneEntries(entries []zoneEntry) ([]map[string]any, map[string]*Library) { + zones := make([]map[string]any, len(entries)) + libByZone := make(map[string]*Library, len(entries)) + for i, e := range entries { + zones[i] = e.zone + libByZone[e.lib.zoneID] = e.lib + } + return zones, libByZone +} + +// batchCheckForChanges checks all zones for changes in a single API call +// Each zone with a syncToken gets checked; zones without tokens are skipped +func (ps *PhotosService) batchCheckForChanges(ctx context.Context, libs map[string]*Library) { + // Group zones by area for separate API calls (private and shared use different endpoints) + byArea := make(map[string][]zoneEntry) + for _, lib := range libs { + lib.deltaMu.Lock() + hasPending := lib.pendingDelta != nil + lib.deltaMu.Unlock() + if hasPending { + continue + } + token := lib.readSyncToken() + if token == "" { + continue + } + zone := map[string]any{ + "zoneID": lib.zoneIDMap(), + "desiredKeys": changesZoneDesiredKeys, + "syncToken": token, + } + byArea[lib.area] = append(byArea[lib.area], zoneEntry{zone: zone, lib: lib}) + } + + for area, entries := range byArea { + zones, libByZone := flattenZoneEntries(entries) + var response changesZoneResponse + if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil { + fs.Debugf(nil, "iclouddrive photos: batch delta check (%s) failed: %v", area, err) + continue + } + for _, zone := range response.Zones { + lib := libByZone[zone.ZoneID.ZoneName] + if lib == nil { + continue + } + lib.deltaMu.Lock() + if len(zone.Records) == 0 && !zone.MoreComing { + fs.Debugf(nil, "iclouddrive photos: zone %s unchanged, using cached listings", lib.zoneID) + lib.saveSyncToken(zone.SyncToken) + lib.cacheValid.Store(true) + } else { + lib.bufferDelta(zone.Records, zone.SyncToken, zone.MoreComing) + } + lib.deltaMu.Unlock() + } + } +} + +// PollForChanges checks all zones for changes in a single API call using +// separate notification tokens, returns zone names that have been modified +// Used by ChangeNotify - does not consume or interfere with listing delta sync +func (ps *PhotosService) PollForChanges(ctx context.Context) []string { + ps.mu.Lock() + libs := make([]*Library, 0, len(ps.libraries)) + for _, lib := range ps.libraries { + libs = append(libs, lib) + } + ps.mu.Unlock() + + // Group zones by area for separate API calls + byArea := make(map[string][]zoneEntry) + for _, lib := range libs { + lib.deltaMu.Lock() + token := lib.notifyToken + lib.deltaMu.Unlock() + if token == "" { + token = lib.readSyncToken() + } + if token == "" { + continue + } + byArea[lib.area] = append(byArea[lib.area], zoneEntry{ + zone: map[string]any{ + "zoneID": lib.zoneIDMap(), + "desiredKeys": changesZoneDesiredKeys, + "syncToken": token, + }, + lib: lib, + }) + } + + var changed []string + for area, entries := range byArea { + zones, libByZone := flattenZoneEntries(entries) + var response changesZoneResponse + if err := ps.requestForArea(ctx, area, "changes/zone", map[string]any{"zones": zones}, &response); err != nil { + continue + } + + for _, zone := range response.Zones { + lib := libByZone[zone.ZoneID.ZoneName] + if lib == nil { + continue + } + lib.deltaMu.Lock() + lib.notifyToken = zone.SyncToken + lib.deltaMu.Unlock() + + if len(zone.Records) == 0 && !zone.MoreComing { + continue + } + + // Drain remaining pages per zone individually + for zone.MoreComing { + lib.deltaMu.Lock() + token := lib.notifyToken + lib.deltaMu.Unlock() + var next changesZoneResponse + body := map[string]any{"zones": []map[string]any{{ + "zoneID": lib.zoneIDMap(), + "desiredKeys": changesZoneDesiredKeys, + "syncToken": token, + }}} + if err := lib.request(ctx, "changes/zone", body, &next); err != nil { + break + } + if len(next.Zones) == 0 { + break + } + lib.deltaMu.Lock() + lib.notifyToken = next.Zones[0].SyncToken + lib.deltaMu.Unlock() + zone.MoreComing = next.Zones[0].MoreComing + } + + changed = append(changed, lib.zoneID) + fs.Debugf(nil, "iclouddrive photos: ChangeNotify detected changes in zone %s", lib.zoneID) + } + } + return changed +} + +// readSyncToken loads the sync token from disk +func (lib *Library) readSyncToken() string { + data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken")) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +// changesZoneResponse is the structure returned by changes/zone +type changesZoneResponse struct { + Zones []changesZoneResult `json:"zones"` +} + +// changesZoneResult is a single zone entry within a changesZoneResponse +type changesZoneResult struct { + ZoneID struct { + ZoneName string `json:"zoneName"` + } `json:"zoneID"` + Records []json.RawMessage `json:"records"` + SyncToken string `json:"syncToken"` + MoreComing bool `json:"moreComing"` +} + +// changesZoneDesiredKeys are the fields requested from changes/zone for +// delta sync classification into smart albums +var changesZoneDesiredKeys = []string{ + // CPLMaster fields + "filenameEnc", "itemType", "resOriginalRes", "resOriginalWidth", "resOriginalHeight", + "resOriginalFileType", "resOriginalVidComplRes", + "resOriginalAltRes", "resOriginalAltFileType", + // CPLAsset classification + metadata fields + "masterRef", "assetDate", "addedDate", + "isFavorite", "isHidden", "isDeleted", "assetSubtype", "assetSubtypeV2", "burstId", + "adjustmentRenderType", + "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType", + "resVidFullRes", "resVidFullFileType", + // CPLContainerRelation field for user album membership invalidation + "containerId", +} + +// changesZoneBody builds the request body for a changes/zone call +func (lib *Library) changesZoneBody(syncToken string) map[string]any { + zone := map[string]any{ + "zoneID": lib.zoneIDMap(), + "desiredKeys": changesZoneDesiredKeys, + } + if syncToken != "" { + zone["syncToken"] = syncToken + } + return map[string]any{"zones": []map[string]any{zone}} +} + +// deltaParseResult holds the classified output of parseDeltaRecords +type deltaParseResult struct { + deletedIDs map[string]bool + newMasters map[string]*photoRecord + newAssets map[string]*photoRecord + changedAlbumRecords map[string]bool + albumMetadataChanged bool + hasAssetOnlyUpdates bool +} + +func relationAlbumRecordFromRecordName(recordName string) (string, bool) { + parts := strings.SplitN(recordName, "-IN-", 2) + if len(parts) != 2 || parts[1] == "" { + return "", false + } + return parts[1], true +} + +// shouldInvalidate returns true if an album's cache should be invalidated +// rather than incrementally updated from this delta +func (r *deltaParseResult) shouldInvalidate(recordName string, isSmart bool) bool { + return (!isSmart && r.changedAlbumRecords[recordName]) || + (isSmart && r.hasAssetOnlyUpdates) +} + +// parseDeltaRecords classifies raw delta records from changes/zone into +// deletions, new masters/assets, album membership changes, and metadata flags +func parseDeltaRecords(records []json.RawMessage) *deltaParseResult { + r := &deltaParseResult{ + deletedIDs: map[string]bool{}, + newMasters: map[string]*photoRecord{}, + newAssets: map[string]*photoRecord{}, + changedAlbumRecords: map[string]bool{}, + } + + for _, raw := range records { + var header struct { + RecordName string `json:"recordName"` + RecordType string `json:"recordType"` + Deleted bool `json:"deleted"` + } + if err := json.Unmarshal(raw, &header); err != nil { + continue + } + if header.Deleted && header.RecordType == "" { + if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok { + r.changedAlbumRecords[albumRecord] = true + continue + } + } + switch header.RecordType { + case "CPLMaster": + if header.Deleted { + r.deletedIDs[header.RecordName] = true + } else { + var rec photoRecord + if err := json.Unmarshal(raw, &rec); err == nil { + r.newMasters[rec.RecordName] = &rec + } + } + case "CPLAsset": + if header.Deleted { + // Asset deletion - mark both the asset itself and its master for removal + // The asset ID is needed because edited entries use asset.RecordName as ID + // (not master), so filtering by master alone leaves ghost -edited entries + r.deletedIDs[header.RecordName] = true + var rec photoRecord + if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil { + r.deletedIDs[rec.Fields.MasterRef.Value.RecordName] = true + } + } else { + var rec photoRecord + if err := json.Unmarshal(raw, &rec); err == nil && rec.Fields.MasterRef != nil { + r.newAssets[rec.Fields.MasterRef.Value.RecordName] = &rec + } + } + case "CPLAlbum": + r.albumMetadataChanged = true + case "CPLContainerRelation": + var rel struct { + Fields struct { + ContainerID *struct { + Value string `json:"value"` + } `json:"containerId"` + } `json:"fields"` + } + if err := json.Unmarshal(raw, &rel); err == nil { + if rel.Fields.ContainerID != nil && rel.Fields.ContainerID.Value != "" { + r.changedAlbumRecords[rel.Fields.ContainerID.Value] = true + continue + } + } + // Deleted relation records and some changes/zone entries omit fields, + // the deterministic recordName still encodes the target album record + if albumRecord, ok := relationAlbumRecordFromRecordName(header.RecordName); ok { + r.changedAlbumRecords[albumRecord] = true + } else if header.Deleted { + fs.Debugf(nil, "iclouddrive photos: deleted CPLContainerRelation %q has unexpected recordName format", header.RecordName) + } + } + } + + // Detect asset-only metadata updates (favorite/hide/soft-delete toggle) + for masterID := range r.newAssets { + if _, hasMaster := r.newMasters[masterID]; !hasMaster && !r.deletedIDs[masterID] { + r.hasAssetOnlyUpdates = true + break + } + } + + return r +} + +// applyPendingDelta consumes a buffered delta and applies it to album caches +// Called from GetPhotos after albums are guaranteed populated +// Returns true if cache is current (no pending delta, or delta applied successfully) +func (lib *Library) applyPendingDelta(ctx context.Context) bool { + lib.deltaMu.Lock() + defer lib.deltaMu.Unlock() + + pending := lib.pendingDelta + if pending == nil { + return true // nothing pending, cache is current + } + failPendingDelta := func() bool { + lib.pendingDelta = nil + lib.cacheValid.Store(false) + return false + } + + // Verify albums are populated - if not (e.g. eager album invalidation + // cleared the map before GetAlbums ran), clear pendingDelta so + // checkForChanges can re-detect it on the next call. SyncToken was not + // advanced so the same delta will be returned by changes/zone + lib.mu.Lock() + hasAlbums := len(lib.albums) > 0 + lib.mu.Unlock() + if !hasAlbums { + lib.pendingDelta = nil + return false + } + + // Collect all delta records (first page from buffer + remaining pages from API) + allRecords := pending.records + syncToken := pending.syncToken + moreComing := pending.moreComing + for moreComing { + var response changesZoneResponse + if err := lib.request(ctx, "changes/zone", lib.changesZoneBody(syncToken), &response); err != nil { + return failPendingDelta() + } + if len(response.Zones) == 0 { + return failPendingDelta() + } + allRecords = append(allRecords, response.Zones[0].Records...) + syncToken = response.Zones[0].SyncToken + moreComing = response.Zones[0].MoreComing + } + + result := parseDeltaRecords(allRecords) + + // Build new Photo entries from delta master+asset pairs + var addedPhotos []*Photo + for masterID, master := range result.newMasters { + built := buildPhotos(master, result.newAssets[masterID]) + addedPhotos = append(addedPhotos, built...) + } + + fs.Debugf(nil, "iclouddrive photos: zone %s delta: %d deleted, %d added, %d album membership changes from %d records", + lib.zoneID, len(result.deletedIDs), len(addedPhotos), len(result.changedAlbumRecords), len(allRecords)) + + // Apply delta to each album's disk cache + // Pre-resolve cache dir from lib to avoid re-acquiring ps.mu + // under deltaMu (lock ordering: deltaMu must not precede ps.mu) + cacheDir := lib.zoneCacheDir() + lib.mu.Lock() + albums := make([]*Album, 0, len(lib.albums)) + for _, album := range flattenAlbumTree(nil, lib.albums) { + if album.ObjectType != "" { + albums = append(albums, album) + } + } + lib.mu.Unlock() + + for _, album := range albums { + cached, ok := album.loadDiskCacheFrom(cacheDir) + if !ok { + continue + } + + _, isSmart := SmartAlbums[album.Name] + + // Skip albums that will be invalidated below (avoids stale-data + // window for concurrent readers and wasted disk I/O) + if result.shouldInvalidate(album.RecordName, isSmart) { + continue + } + + // Remove deleted/changed entries + filtered := make([]*Photo, 0, len(cached)) + for _, p := range cached { + if !result.deletedIDs[p.ID] { + filtered = append(filtered, p) + } + } + + // Route new photos to smart albums based on classifySmartAlbums() + if isSmart { + for _, p := range addedPhotos { + for _, sa := range p.SmartAlbums { + if sa == album.Name { + filtered = append(filtered, p) + break + } + } + } + } + + album.saveDiskCacheTo(cacheDir, filtered) + + // Update in-memory cache if populated + album.mu.Lock() + if album.photoCache != nil { + // Deep copy before dedup so shared *Photo pointers across albums + // don't get cross-contaminated by filename suffix mutations + deduped := make([]*Photo, len(filtered)) + for i, p := range filtered { + cp := *p + deduped[i] = &cp + } + deduplicateFilenames(deduped) + album.photoCache = buildPhotoCache(deduped) + } + album.mu.Unlock() + } + + // Invalidate caches for albums affected by membership or metadata changes + if len(result.changedAlbumRecords) > 0 || result.hasAssetOnlyUpdates { + lib.mu.Lock() + var invalidated []*Album + for _, album := range flattenAlbumTree(nil, lib.albums) { + _, isSmart := SmartAlbums[album.Name] + if result.shouldInvalidate(album.RecordName, isSmart) { + invalidated = append(invalidated, album) + } + } + lib.mu.Unlock() + for _, album := range invalidated { + album.mu.Lock() + album.photoCache = nil + album.mu.Unlock() + if album.ObjectType != "" { + _ = os.Remove(filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json")) + } + fs.Debugf(nil, "iclouddrive photos: invalidated album %q cache", album.Name) + } + } + + // Album metadata change (CPLAlbum created/renamed/deleted) - clear album + // list so GetAlbums re-fetches from API on next call + if result.albumMetadataChanged { + lib.invalidateAlbumCache() + fs.Debugf(nil, "iclouddrive photos: zone %s album metadata changed, will re-fetch album list", lib.zoneID) + } + + lib.pendingDelta = nil + lib.saveSyncToken(syncToken) + return true +} + +// saveSyncToken persists the zone sync token to disk via atomic rename +func (lib *Library) saveSyncToken(token string) { + dir := lib.zoneCacheDir() + if err := os.MkdirAll(dir, 0700); err != nil { + fs.Debugf(nil, "iclouddrive photos: failed to create cache dir: %v", err) + return + } + if err := atomicWriteFile(filepath.Join(dir, "syncToken"), []byte(token)); err != nil { + fs.Debugf(nil, "iclouddrive photos: failed to write sync token: %v", err) + } +} + +// clearDiskCache removes all cached album data and sync token for this zone +func (lib *Library) clearDiskCache() { + dir := lib.zoneCacheDir() + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + _ = os.Remove(filepath.Join(dir, e.Name())) + } +} + +// request makes an API call routed through the album's library area +func (album *Album) request(ctx context.Context, endpoint string, data, response any) error { + if album.lib != nil { + return album.lib.request(ctx, endpoint, data, response) + } + return fmt.Errorf("album %q has no library", album.Name) +} + +// zoneIDMap returns the full zoneID for this album's zone +func (album *Album) zoneIDMap() map[string]any { + if album.lib == nil { + return map[string]any{} + } + return album.lib.zoneIDMap() +} + +// loadDiskCacheFrom loads cached photo data from a specific cache directory +func (album *Album) loadDiskCacheFrom(cacheDir string) ([]*Photo, bool) { + cacheFile := filepath.Join(cacheDir, albumCacheKey(album.ObjectType)+".json") + data, err := os.ReadFile(cacheFile) + if err != nil { + return nil, false + } + var photos []*Photo + if err := json.Unmarshal(data, &photos); err != nil { + return nil, false + } + return photos, true +} + +// saveDiskCache persists photo data to disk for delta sync via atomic rename +func (album *Album) saveDiskCache(photos []*Photo) { + if album.ObjectType == "" || album.lib == nil { + return + } + album.saveDiskCacheTo(album.lib.zoneCacheDir(), photos) +} + +// saveDiskCacheTo persists photo data to a specific cache directory +func (album *Album) saveDiskCacheTo(dir string, photos []*Photo) { + saveJSONCache(dir, albumCacheKey(album.ObjectType)+".json", photos) +} + +// countQuery builds a HyperionIndexCountLookup query for a single object type +func countQuery(objectType string, zoneID map[string]any) map[string]any { + return map[string]any{ + "resultsLimit": 1, + "query": map[string]any{ + "filterBy": map[string]any{ + "fieldName": "indexCountID", + "fieldValue": map[string]any{"type": "STRING_LIST", "value": []string{objectType}}, + "comparator": "IN", + }, + "recordType": recordTypeCountIndex, + }, + "zoneWide": true, + "zoneID": zoneID, + } +} + +// countBatchResponse is the response shape for batched count queries +type countBatchResponse struct { + Batch []struct { + Records []struct { + Fields struct { + ItemCount struct { + Value int64 `json:"value"` + } `json:"itemCount"` + } `json:"fields"` + } `json:"records"` + } `json:"batch"` +} + +// toCounts maps an ordered list of names to their counts from the batch response +func (r *countBatchResponse) toCounts(names []string) map[string]int64 { + counts := make(map[string]int64, len(names)) + for i, name := range names { + if i < len(r.Batch) && len(r.Batch[i].Records) > 0 { + counts[name] = r.Batch[i].Records[0].Fields.ItemCount.Value + } + } + return counts +} + +// GetLibraryAlbumCounts returns the album count for each library in a single +// batched request, keyed by zone name (e.g. "PrimarySync") +func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]int64, error) { + ps.mu.Lock() + type libEntry struct { + name string + lib *Library + } + byArea := make(map[string][]libEntry) + for name, lib := range ps.libraries { + byArea[lib.area] = append(byArea[lib.area], libEntry{name: name, lib: lib}) + } + ps.mu.Unlock() + + counts := make(map[string]int64) + for area, entries := range byArea { + var batch []map[string]any + var order []string + for _, e := range entries { + order = append(order, e.name) + batch = append(batch, countQuery(recordTypeAlbum, e.lib.zoneIDMap())) + } + var response countBatchResponse + if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil { + return nil, fmt.Errorf("failed to get library album counts: %w", err) + } + for k, v := range response.toCounts(order) { + counts[k] = v + } + } + return counts, nil +} + +// CloudKit field types for record deserialization +type ckStringField struct { + Value string `json:"value"` + Type string `json:"type,omitempty"` // present on filenameEnc (ENCRYPTED_BYTES vs STRING) +} + +type ckIntField struct { + Value int `json:"value"` +} + +type ckTimestampField struct { + Value int64 `json:"value"` +} + +type ckResourceField struct { + Value struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + } `json:"value"` +} + +type ckBoolField struct { + Value bool `json:"value"` +} + +type ckReferenceField struct { + Value struct { + RecordName string `json:"recordName"` + } `json:"value"` +} + +// photoRecord represents a CloudKit record (CPLAsset or CPLMaster) +type photoRecord struct { + RecordName string `json:"recordName"` + RecordType string `json:"recordType"` + Fields struct { + FilenameEnc *ckStringField `json:"filenameEnc,omitempty"` + ItemType *ckStringField `json:"itemType,omitempty"` + ResOriginalRes *ckResourceField `json:"resOriginalRes,omitempty"` + ResOriginalWidth *ckIntField `json:"resOriginalWidth,omitempty"` + ResOriginalHeight *ckIntField `json:"resOriginalHeight,omitempty"` + ResOriginalFileType *ckStringField `json:"resOriginalFileType,omitempty"` + ResOriginalVidComplRes *ckResourceField `json:"resOriginalVidComplRes,omitempty"` + ResOriginalAltRes *ckResourceField `json:"resOriginalAltRes,omitempty"` + ResOriginalAltFileType *ckStringField `json:"resOriginalAltFileType,omitempty"` + MasterRef *ckReferenceField `json:"masterRef,omitempty"` + AssetDate *ckTimestampField `json:"assetDate,omitempty"` + AddedDate *ckTimestampField `json:"addedDate,omitempty"` + IsFavorite *ckIntField `json:"isFavorite,omitempty"` + IsHidden *ckIntField `json:"isHidden,omitempty"` + AssetSubtype *ckIntField `json:"assetSubtype,omitempty"` + AssetSubtypeV2 *ckIntField `json:"assetSubtypeV2,omitempty"` + BurstID *ckStringField `json:"burstId,omitempty"` + AdjustmentRenderType *ckIntField `json:"adjustmentRenderType,omitempty"` + IsDeleted *ckIntField `json:"isDeleted,omitempty"` + AdjustmentType *ckStringField `json:"adjustmentType,omitempty"` + ResJPEGFullRes *ckResourceField `json:"resJPEGFullRes,omitempty"` + ResJPEGFullFileType *ckStringField `json:"resJPEGFullFileType,omitempty"` + ResVidFullRes *ckResourceField `json:"resVidFullRes,omitempty"` + ResVidFullFileType *ckStringField `json:"resVidFullFileType,omitempty"` + } `json:"fields"` +} + +// classifySmartAlbums determines which smart albums a photo belongs to +// based on CPLMaster and CPLAsset fields from CloudKit +func classifySmartAlbums(master *photoRecord, asset *photoRecord) []string { + isVideo := false + if master.Fields.ResOriginalFileType != nil { + uti := master.Fields.ResOriginalFileType.Value + isVideo = uti == "public.mpeg-4" || uti == "com.apple.quicktime-movie" + } + + var subtype, subtypeV2, favorite, hidden, deleted int + if asset != nil { + if asset.Fields.AssetSubtype != nil { + subtype = asset.Fields.AssetSubtype.Value + } + if asset.Fields.AssetSubtypeV2 != nil { + subtypeV2 = asset.Fields.AssetSubtypeV2.Value + } + if asset.Fields.IsFavorite != nil { + favorite = asset.Fields.IsFavorite.Value + } + if asset.Fields.IsHidden != nil { + hidden = asset.Fields.IsHidden.Value + } + if asset.Fields.IsDeleted != nil { + deleted = asset.Fields.IsDeleted.Value + } + } + + // Soft-deleted assets (isDeleted=1) go to Recently Deleted only + if deleted == 1 { + return []string{"Recently Deleted"} + } + + var albums []string + if hidden == 0 { + albums = append(albums, "All Photos") + } + if hidden == 1 { + albums = append(albums, "Hidden") + } + if favorite == 1 { + albums = append(albums, "Favorites") + } + if isVideo && subtype == 0 { + albums = append(albums, "Videos") + } + if subtype == subtypeSloMo { + albums = append(albums, "Slo-mo") + } + if subtype == subtypeTimeLapse { + albums = append(albums, "Time-lapse") + } + if subtype == subtypePanorama { + albums = append(albums, "Panoramas") + } + if subtypeV2 == subtypeV2Live { + albums = append(albums, "Live") + } + if subtypeV2 == subtypeV2Screenshot { + albums = append(albums, "Screenshots") + } + if asset != nil && asset.Fields.BurstID != nil && asset.Fields.BurstID.Value != "" { + albums = append(albums, "Bursts") + } + // adjustmentRenderType is a bitmask: PORTRAIT=2, LONG_EXPOSURE=4 + if asset != nil && asset.Fields.AdjustmentRenderType != nil { + art := asset.Fields.AdjustmentRenderType.Value + if art&adjustPortrait != 0 { + albums = append(albums, "Portrait") + } + if art&adjustLongExposure != 0 { + albums = append(albums, "Long Exposure") + } + } + // Animated (GIFs) detected by file type on master record + if master.Fields.ResOriginalFileType != nil && master.Fields.ResOriginalFileType.Value == "com.compuserve.gif" { + albums = append(albums, "Animated") + } + // Selfies: no reliable field available from delta records - server query handles it + return albums +} + +// deduplicateFilenames renames ALL photos with colliding filenames by appending +// the full masterID (CloudKit recordName) before the extension. Every duplicate +// gets the suffix so filenames are stable when photos are added or removed +// Unique filenames are untouched. Collision-free by construction since CloudKit +// recordNames are unique. Same pattern as googlephotos which embeds the full +// media item ID ({55+ chars}) in every filename +func deduplicateFilenames(photos []*Photo) { + counts := make(map[string]int, len(photos)) + for _, p := range photos { + counts[p.Filename]++ + } + for _, p := range photos { + if counts[p.Filename] <= 1 { + continue + } + ext := path.Ext(p.Filename) + base := strings.TrimSuffix(p.Filename, ext) + p.Filename = base + "_" + p.ID + ext + } +} + +// buildPhotos creates Photo entries from a CPLMaster record and its paired CPLAsset +// Returns 1-2 entries: the photo itself, plus a .MOV companion for Live Photos +func buildPhotos(master *photoRecord, asset *photoRecord) []*Photo { + photo := &Photo{ID: master.RecordName} + + if master.Fields.FilenameEnc != nil { + if master.Fields.FilenameEnc.Type == "STRING" { + photo.Filename = norm.NFC.String(master.Fields.FilenameEnc.Value) + } else if decoded, err := base64.StdEncoding.DecodeString(master.Fields.FilenameEnc.Value); err == nil { + photo.Filename = norm.NFC.String(string(decoded)) + } + } + // Fallback: synthesize filename from recordName + itemType UTI when filenameEnc is missing + if photo.Filename == "" && master.Fields.ItemType != nil { + if ext, ok := utiExtensions[master.Fields.ItemType.Value]; ok { + photo.Filename = master.RecordName + ext + } + } + + if master.Fields.ResOriginalRes != nil { + photo.Size = master.Fields.ResOriginalRes.Value.Size + } + + if master.Fields.ResOriginalWidth != nil { + photo.Width = master.Fields.ResOriginalWidth.Value + } + if master.Fields.ResOriginalHeight != nil { + photo.Height = master.Fields.ResOriginalHeight.Value + } + + var liveVideoSize int64 + var hasLiveVideo bool + if master.Fields.ResOriginalVidComplRes != nil && master.Fields.ResOriginalVidComplRes.Value.DownloadURL != "" { + liveVideoSize = master.Fields.ResOriginalVidComplRes.Value.Size + hasLiveVideo = true + } + + if asset != nil { + if asset.Fields.AssetDate != nil { + photo.AssetDate = asset.Fields.AssetDate.Value + } + if asset.Fields.AddedDate != nil { + photo.AddedDate = asset.Fields.AddedDate.Value + } + photo.IsFavorite = asset.Fields.IsFavorite != nil && asset.Fields.IsFavorite.Value == 1 + photo.IsHidden = asset.Fields.IsHidden != nil && asset.Fields.IsHidden.Value == 1 + } + + photo.SmartAlbums = classifySmartAlbums(master, asset) + + hasDownloadURL := master.Fields.ResOriginalRes != nil && master.Fields.ResOriginalRes.Value.DownloadURL != "" + if !hasDownloadURL || photo.Filename == "" { + return nil + } + + photo.ResourceKey = "resOriginalRes" + result := []*Photo{photo} + + ext := path.Ext(photo.Filename) + stem := strings.TrimSuffix(photo.Filename, ext) + + if hasLiveVideo { + result = append(result, photo.companion(photo.ID, stem+".MOV", "resOriginalVidComplRes", liveVideoSize)) + } + + // Edited photo version (Photos.app adjustments) + // Slo-mo edits are metadata-only (playback speed) with no separate rendered resource + if asset != nil && asset.Fields.AdjustmentType != nil && + asset.Fields.AdjustmentType.Value != "" && + asset.Fields.AdjustmentType.Value != adjustSloMo { + if asset.Fields.ResJPEGFullRes != nil && asset.Fields.ResJPEGFullRes.Value.DownloadURL != "" { + editExt := extFromUTI(asset.Fields.ResJPEGFullFileType, ext) + result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resJPEGFullRes", asset.Fields.ResJPEGFullRes.Value.Size)) + } else if asset.Fields.ResVidFullRes != nil && asset.Fields.ResVidFullRes.Value.DownloadURL != "" { + editExt := extFromUTI(asset.Fields.ResVidFullFileType, ext) + result = append(result, photo.companion(asset.RecordName, stem+"-edited"+editExt, "resVidFullRes", asset.Fields.ResVidFullRes.Value.Size)) + } + } + + // RAW alternative (RAW+JPEG pairs where both originals are stored) + if master.Fields.ResOriginalAltRes != nil && master.Fields.ResOriginalAltRes.Value.DownloadURL != "" { + altExt := extFromUTI(master.Fields.ResOriginalAltFileType, ext) + altFilename := stem + altExt + if strings.EqualFold(altFilename, photo.Filename) { + altFilename = stem + "-alt" + altExt + } + alt := photo.companion(master.RecordName, altFilename, "resOriginalAltRes", master.Fields.ResOriginalAltRes.Value.Size) + alt.Width = photo.Width // same sensor capture, same dimensions + alt.Height = photo.Height // same sensor capture, same dimensions + result = append(result, alt) + } + + return result +} + +// photosDesiredKeys are the fields requested for photo listing +var photosDesiredKeys = []string{ + "resOriginalRes", "resOriginalVidComplRes", "resOriginalFileType", + "resOriginalWidth", "resOriginalHeight", + "resOriginalAltRes", "resOriginalAltFileType", + "filenameEnc", "itemType", "assetDate", "addedDate", "masterRef", + "isFavorite", "isHidden", "isDeleted", + "assetSubtype", "assetSubtypeV2", "burstId", "adjustmentRenderType", + "adjustmentType", "resJPEGFullRes", "resJPEGFullFileType", + "resVidFullRes", "resVidFullFileType", +} + +// fetchPhotoCount returns the photo count for this album via HyperionIndexCountLookup +func (album *Album) fetchPhotoCount(ctx context.Context) (int64, error) { + if album.ObjectType == "" { + return 0, nil + } + var response struct { + Records []struct { + Fields struct { + ItemCount struct { + Value int64 `json:"value"` + } `json:"itemCount"` + } `json:"fields"` + } `json:"records"` + } + if err := album.request(ctx, "records/query", countQuery(album.ObjectType, album.zoneIDMap()), &response); err != nil { + return 0, err + } + if len(response.Records) > 0 { + return response.Records[0].Fields.ItemCount.Value, nil + } + return 0, nil +} + +// parsePhotoRecords extracts Photo entries from a batch of CloudKit records +func parsePhotoRecords(records []photoRecord) []*Photo { + if len(records) == 0 { + return nil + } + half := len(records)/2 + 1 + assetMap := make(map[string]*photoRecord, half) + masters := make([]*photoRecord, 0, half) + for i := range records { + record := &records[i] + switch record.RecordType { + case "CPLAsset": + if record.Fields.MasterRef != nil { + assetMap[record.Fields.MasterRef.Value.RecordName] = record + } + case "CPLMaster": + masters = append(masters, record) + } + } + photos := make([]*Photo, 0, len(masters)) + for _, master := range masters { + built := buildPhotos(master, assetMap[master.RecordName]) + photos = append(photos, built...) + } + return photos +} + +// buildPartitionQuery constructs a CloudKit records/query body for a single +// startRank EQUALS partition, including album-specific direction and filters +func (album *Album) buildPartitionQuery(startRank int) map[string]any { + filters := []map[string]any{ + { + "fieldName": "startRank", + "comparator": "EQUALS", + "fieldValue": map[string]any{"type": "INT64", "value": startRank}, + }, + { + "fieldName": "direction", + "fieldValue": map[string]any{"type": "STRING", "value": album.Direction}, + "comparator": "EQUALS", + }, + } + for _, filter := range album.Filters { + filters = append(filters, map[string]any{ + "fieldName": filter.FieldName, + "comparator": filter.Comparator, + "fieldValue": filter.FieldValue, + }) + } + return map[string]any{ + "query": map[string]any{ + "filterBy": filters, + "recordType": album.ListType, + }, + "resultsLimit": photosQueryLimit, + "desiredKeys": photosDesiredKeys, + "zoneID": album.zoneIDMap(), + } +} + +// fetchPhotosParallel fetches all photos using parallel startRank partitions +// Each partition is one API call with startRank EQUALS, no continuationMarker +// Stride = photosQueryLimit/2 photos per partition (200 records = 100 photos) +func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64) ([]*Photo, string, error) { + stride := photosQueryLimit / 2 // 100 photos per partition + numPartitions := int((totalPhotos + int64(stride) - 1) / int64(stride)) + workers := fs.GetConfig(ctx).Checkers + if workers < 1 { + workers = 8 + } + + fs.Logf(nil, "iclouddrive photos: parallel cold listing %d photos in %d partitions (%d workers)", + totalPhotos, numPartitions, workers) + + type partitionResult struct { + photos []*Photo + syncToken string + recordCount int // raw record count to detect full pages + err error + } + + // Cancel all remaining goroutines on first error + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + results := make([]partitionResult, numPartitions) + sem := make(chan struct{}, workers) + var wg sync.WaitGroup + + for i := 0; i < numPartitions; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + select { + case sem <- struct{}{}: + case <-ctx.Done(): + results[idx] = partitionResult{err: ctx.Err()} + return + } + defer func() { <-sem }() + + startRank := idx * stride + query := album.buildPartitionQuery(startRank) + + var response struct { + Records []photoRecord `json:"records"` + SyncToken string `json:"syncToken"` + } + if err := album.request(ctx, "records/query", query, &response); err != nil { + cancel() // stop remaining partitions + results[idx] = partitionResult{err: fmt.Errorf("partition %d (rank=%d): %w", idx, startRank, err)} + return + } + results[idx] = partitionResult{ + photos: parsePhotoRecords(response.Records), + syncToken: response.SyncToken, + recordCount: len(response.Records), + } + }(i) + } + wg.Wait() + + // Merge results in order + var allPhotos []*Photo + var lastSyncToken string + var lastRecordCount int + for _, r := range results { + if r.err != nil { + return nil, "", r.err + } + allPhotos = append(allPhotos, r.photos...) + if r.syncToken != "" { + lastSyncToken = r.syncToken + } + lastRecordCount = r.recordCount + } + + // Completeness: keep fetching until a partition returns fewer than + // resultsLimit records (partial page = last page). This handles stale + // counts AND count=0 (count query failed - discover all photos here) + needsTailCheck := lastRecordCount >= photosQueryLimit || numPartitions == 0 + nextRank := numPartitions * stride + for needsTailCheck { + fs.Debugf(nil, "iclouddrive photos: fetching tail partition at rank=%d", nextRank) + query := album.buildPartitionQuery(nextRank) + var response struct { + Records []photoRecord `json:"records"` + SyncToken string `json:"syncToken"` + } + if err := album.request(ctx, "records/query", query, &response); err != nil { + return nil, "", fmt.Errorf("tail partition (rank=%d): %w", nextRank, err) + } + lastRecordCount = len(response.Records) + allPhotos = append(allPhotos, parsePhotoRecords(response.Records)...) + if response.SyncToken != "" { + lastSyncToken = response.SyncToken + } + nextRank += stride + needsTailCheck = lastRecordCount >= photosQueryLimit + } + + fs.Logf(nil, "iclouddrive photos: parallel fetch complete, %d photos", len(allPhotos)) + return allPhotos, lastSyncToken, nil +} + +// GetPhotos retrieves photos from this album using parallel partitions with disk cache +func (album *Album) GetPhotos(ctx context.Context) ([]*Photo, error) { + // No library configured - return pre-populated cache (test path) + if album.lib == nil { + album.mu.Lock() + defer album.mu.Unlock() + result := make([]*Photo, 0, len(album.photoCache)) + for _, p := range album.photoCache { + result = append(result, p) + } + return result, nil + } + + // Check for changes, apply any buffered delta, serve from cache + if album.ObjectType != "" { + album.lib.checkForChanges(ctx) + if album.lib.applyPendingDelta(ctx) && album.lib.cacheValid.Load() { + // Serve from in-memory cache if populated (avoids disk I/O + JSON parse + dedup) + album.mu.Lock() + if album.photoCache != nil { + result := make([]*Photo, 0, len(album.photoCache)) + for _, p := range album.photoCache { + result = append(result, p) + } + album.mu.Unlock() + return result, nil + } + album.mu.Unlock() + // Fall back to disk cache + if cached, ok := album.loadDiskCacheFrom(album.lib.zoneCacheDir()); ok { + deduplicateFilenames(cached) + album.mu.Lock() + album.photoCache = buildPhotoCache(cached) + album.mu.Unlock() + fs.Debugf(nil, "iclouddrive photos: %d items from cache for %q", len(cached), album.Name) + return cached, nil + } + } + } + + // Fetch photo count for parallel partition calculation + // If count unavailable, use 0 - the tail-fetch loop handles completeness + var count int64 + if album.ObjectType != "" { + count, _ = album.fetchPhotoCount(ctx) + } + photos, lastSyncToken, err := album.fetchPhotosParallel(ctx, count) + if err != nil { + return nil, err + } + + // Persist original filenames for delta sync (dedup is applied on read) + album.saveDiskCache(photos) + + deduplicateFilenames(photos) + + // Populate filename cache for NewObject lookups + album.mu.Lock() + album.photoCache = buildPhotoCache(photos) + album.mu.Unlock() + if lastSyncToken != "" && album.lib != nil { + album.lib.saveSyncToken(lastSyncToken) + } + + return photos, nil +} + +// GetPhotoByName looks up a photo by filename, using cache if available +// CloudKit has no filterBy on filename fields - the only queryable fields +// are rank, date, smartAlbum, etc. Apple's own icloud.com UI paginates +// the full album and indexes client-side. On cache miss we must enumerate +// the entire album via GetPhotos before lookup +func (album *Album) GetPhotoByName(ctx context.Context, filename string) (*Photo, error) { + album.mu.Lock() + if album.photoCache != nil { + photo, exists := album.photoCache[filename] + album.mu.Unlock() + if exists { + return photo, nil + } + return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name) + } + album.mu.Unlock() + + // Cache miss - fetch all photos to populate cache + photos, err := album.GetPhotos(ctx) + if err != nil { + return nil, err + } + for _, photo := range photos { + if photo.Filename == filename { + return photo, nil + } + } + return nil, fmt.Errorf("photo %q not found in album %q", filename, album.Name) +} + +// GetAlbumCounts returns photo counts for all albums in a single batch request +func (lib *Library) GetAlbumCounts(ctx context.Context) (map[string]int64, error) { + // Snapshot under lock to avoid racing with GetAlbums + type albumEntry struct { + name string + objectType string + } + lib.mu.Lock() + entries := make([]albumEntry, 0, len(lib.albums)) + for name, album := range lib.albums { + if album.ObjectType == "" { + continue // skip folders (albumType=3), they have no photo count + } + entries = append(entries, albumEntry{name: name, objectType: album.ObjectType}) + } + lib.mu.Unlock() + + if len(entries) == 0 { + return nil, nil + } + + zoneIDAny := lib.zoneIDMap() + + var batch []map[string]any + var albumOrder []string + for _, entry := range entries { + albumOrder = append(albumOrder, entry.name) + batch = append(batch, countQuery(entry.objectType, zoneIDAny)) + } + var response countBatchResponse + if err := lib.request(ctx, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil { + return nil, err + } + return response.toCounts(albumOrder), nil +} + +// resolveZone returns the area and full zoneID for a zone name, +// using the library metadata if available or falling back to private +func (ps *PhotosService) resolveZone(zoneName string) (area string, zoneID map[string]any) { + ps.mu.Lock() + lib := ps.libraries[zoneName] + ps.mu.Unlock() + if lib != nil { + return lib.area, lib.zoneIDMap() + } + return areaPrivate, map[string]any{"zoneName": zoneName} +} + +// LookupDownloadURL fetches a fresh download URL for a record +// recordName is the CPLMaster or CPLAsset recordName depending on the resource +// resourceKey selects which resource to look up (e.g. "resOriginalRes", +// "resOriginalVidComplRes" for Live Photo video, "resJPEGFullRes" for edited) +func (ps *PhotosService) LookupDownloadURL(ctx context.Context, recordName, zone, resourceKey string) (string, error) { + area, zoneID := ps.resolveZone(zone) + + query := map[string]any{ + "records": []map[string]any{ + {"recordName": recordName}, + }, + "zoneID": zoneID, + } + + var response struct { + Records []json.RawMessage `json:"records"` + } + + if err := ps.requestForArea(ctx, area, "records/lookup", query, &response); err != nil { + return "", fmt.Errorf("failed to look up record %q: %w", recordName, err) + } + + if len(response.Records) == 0 { + return "", fmt.Errorf("no records in lookup response for %q", recordName) + } + + // Parse fields as raw JSON to extract the requested resource key + var record struct { + Fields map[string]json.RawMessage `json:"fields"` + } + if err := json.Unmarshal(response.Records[0], &record); err != nil { + return "", fmt.Errorf("failed to parse lookup response: %w", err) + } + + rawField, exists := record.Fields[resourceKey] + if !exists { + return "", fmt.Errorf("no %q field in record %q", resourceKey, recordName) + } + + var res struct { + Value struct { + DownloadURL string `json:"downloadURL"` + } `json:"value"` + } + if err := json.Unmarshal(rawField, &res); err != nil { + return "", fmt.Errorf("failed to parse %q field: %w", resourceKey, err) + } + if res.Value.DownloadURL == "" { + return "", fmt.Errorf("no download URL for %q in record %q", resourceKey, recordName) + } + + return res.Value.DownloadURL, nil +} + +// checkIndexingState warns if the iCloud Photo Library is still indexing +func (ps *PhotosService) checkIndexingState(ctx context.Context, zoneName string) { + area, zoneID := ps.resolveZone(zoneName) + + query := map[string]any{ + "query": map[string]any{"recordType": "CheckIndexingState"}, + "zoneID": zoneID, + } + + var response struct { + Records []struct { + Fields struct { + State struct { + Value string `json:"value"` + } `json:"state"` + } `json:"fields"` + } `json:"records"` + } + + if err := ps.requestForArea(ctx, area, "records/query", query, &response); err != nil { + fs.Logf(nil, "iclouddrive photos: could not check indexing state: %v", err) + return + } + + if len(response.Records) == 0 || response.Records[0].Fields.State.Value != indexingStateReady { + fs.Logf(nil, "iclouddrive photos: library is still indexing, listings may be incomplete") + } +} + +// requestWithReauth makes a CloudKit request with pacer retry and reauth on 401/421 +func (ps *PhotosService) requestWithReauth(ctx context.Context, makeOpts func() rest.Opts, data, response any) error { + reauthDone := false + return ps.pacer.Call(func() (bool, error) { + resp, err := ps.client.Session.Request(ctx, makeOpts(), data, response) + if !reauthDone && err != nil && resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 421) { + reauthDone = true + if authErr := ps.client.Authenticate(ctx); authErr != nil { + return false, authErr + } + if ps.client.Session.Requires2FA() { + return false, errors.New("trust token expired, please reauth") + } + resp, err = ps.client.Session.Request(ctx, makeOpts(), data, response) + } + return ps.shouldRetry(ctx, resp, err) + }) +} + +// requestForArea makes a request to the given area (private or shared) endpoint +func (ps *PhotosService) requestForArea(ctx context.Context, area, endpoint string, data, response any) error { + rootURL := fmt.Sprintf("%s/%s/%s?remapEnums=true&getCurrentSyncToken=true", ps.endpoint, area, endpoint) + + return ps.requestWithReauth(ctx, func() rest.Opts { + return rest.Opts{ + Method: "POST", + RootURL: rootURL, + ExtraHeaders: ps.client.Session.GetHeaders(map[string]string{"Content-Type": "text/plain"}), // text/plain matches icloud.com (CORS preflight bypass) + } + }, data, response) +} diff --git a/backend/iclouddrive/api/photos_test.go b/backend/iclouddrive/api/photos_test.go new file mode 100644 index 000000000..59ec73168 --- /dev/null +++ b/backend/iclouddrive/api/photos_test.go @@ -0,0 +1,1285 @@ +package api + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/rest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setTestCacheDir(t *testing.T) string { + t.Helper() + oldCacheDir := config.GetCacheDir() + cacheDir := t.TempDir() + require.NoError(t, config.SetCacheDir(cacheDir)) + t.Cleanup(func() { + _ = config.SetCacheDir(oldCacheDir) + }) + return cacheDir +} + +func newHTTPTestPhotosService(t *testing.T, remoteName string, handler http.HandlerFunc) *PhotosService { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + session := NewSession() + session.srv = rest.NewClient(server.Client()) + + return &PhotosService{ + client: &Client{ + remoteName: remoteName, + Session: session, + }, + endpoint: server.URL + "/database/1/com.apple.photos.cloud/production", + pacer: fs.NewPacer(context.Background(), pacer.NewDefault()), + shouldRetry: func(ctx context.Context, resp *http.Response, err error) (bool, error) { return false, err }, + libraries: make(map[string]*Library), + } +} + +func writeJSON(t *testing.T, w http.ResponseWriter, v any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + panic(err) + } +} + +func readJSONBody(r *http.Request, out any) error { + data, err := io.ReadAll(r.Body) + if err != nil { + return err + } + return json.Unmarshal(data, out) +} + +func testAlbumRecordJSON(name, recordName string, albumType int, parentID string) map[string]any { + fields := map[string]any{ + "albumNameEnc": map[string]any{"value": base64.StdEncoding.EncodeToString([]byte(name))}, + } + if albumType != 0 { + fields["albumType"] = map[string]any{"value": albumType} + } + if parentID != "" { + fields["parentId"] = map[string]any{"value": parentID} + } + return map[string]any{ + "recordName": recordName, + "fields": fields, + } +} + +func newUserAlbumForTest(lib *Library, name, recordName string) *Album { + album := lib.newUserAlbum(name, recordName) + album.lib = lib + return album +} + +func TestGetPhotos_DoesNotServeStaleCacheOnPagedDeltaFailure(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + var changesZoneCalls atomic.Int32 + + ps := newHTTPTestPhotosService(t, "paged-delta-failure", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/database/1/com.apple.photos.cloud/production/private/changes/zone": + if changesZoneCalls.Add(1) > 1 { + http.Error(w, "paged delta failure", http.StatusInternalServerError) + return + } + writeJSON(t, w, map[string]any{ + "zones": []map[string]any{{ + "zoneID": map[string]any{"zoneName": "PrimarySync"}, + "records": []map[string]any{{"recordName": "asset1-IN-album-record", "deleted": true}}, + "syncToken": "mid-token", + "moreComing": true, + }}, + }) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + }) + + lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}} + album := newUserAlbumForTest(lib, "Exported", "album-record") + lib.albums[album.Name] = album + album.saveDiskCache([]*Photo{{ID: "stale-id", Filename: "stale.jpg"}}) + require.NoError(t, os.WriteFile(filepath.Join(lib.zoneCacheDir(), "syncToken"), []byte("old-token"), 0600)) + + photos, err := album.GetPhotos(ctx) + assert.Error(t, err) + assert.Nil(t, photos) + assert.Nil(t, lib.pendingDelta, "failed delta apply must not leave pendingDelta stuck forever") +} + +func TestGetLibraries_RediscoversZonesWhenCacheExists(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + var changesDatabaseCalls atomic.Int32 + + ps := newHTTPTestPhotosService(t, "library-rediscovery", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/database/1/com.apple.photos.cloud/production/private/changes/database" { + changesDatabaseCalls.Add(1) + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + return + } + if r.URL.Path == "/database/1/com.apple.photos.cloud/production/shared/changes/database" { + changesDatabaseCalls.Add(1) + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + return + } + http.Error(w, "unexpected request", http.StatusInternalServerError) + }) + + saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}}) + + libs, err := ps.GetLibraries(ctx) + require.NoError(t, err) + assert.Contains(t, libs, "PrimarySync") + assert.Contains(t, libs, "SharedSync-test") + assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "must rediscover zones even when cache exists") +} + +func TestGetLibraries_RediscoveryPreservesBufferedDelta(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + + ps := newHTTPTestPhotosService(t, "library-merge", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/database/1/com.apple.photos.cloud/production/private/changes/zone": + writeJSON(t, w, map[string]any{ + "zones": []map[string]any{{ + "zoneID": map[string]any{"zoneName": "PrimarySync"}, + "records": []map[string]any{{"recordName": "asset123-IN-album-uuid", "deleted": true}}, + "syncToken": "new-token", + "moreComing": false, + }}, + }) + case "/database/1/com.apple.photos.cloud/production/private/changes/database": + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + case "/database/1/com.apple.photos.cloud/production/shared/changes/database": + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-test", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + }) + + saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}}) + require.NoError(t, os.MkdirAll(filepath.Join(ps.client.CacheDir(), "PrimarySync"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(ps.client.CacheDir(), "PrimarySync", "syncToken"), []byte("cached-token"), 0600)) + + libs, err := ps.GetLibraries(ctx) + require.NoError(t, err) + primary := libs["PrimarySync"] + require.NotNil(t, primary) + require.NotNil(t, primary.pendingDelta, "rediscovery must not discard pending delta buffered from cached zone state") + assert.Contains(t, libs, "SharedSync-test") +} + +func TestGetLibraries_RefreshesInMemoryLibraries(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + var changesDatabaseCalls atomic.Int32 + + ps := newHTTPTestPhotosService(t, "library-refresh-memory", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/database/1/com.apple.photos.cloud/production/private/changes/database": + changesDatabaseCalls.Add(1) + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + case "/database/1/com.apple.photos.cloud/production/shared/changes/database": + changesDatabaseCalls.Add(1) + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "SharedSync-live", "ownerRecordName": "other", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + }) + + ps.libraries = map[string]*Library{ + "PrimarySync": {service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: make(map[string]*Album)}, + } + + libs, err := ps.GetLibraries(ctx) + require.NoError(t, err) + assert.Contains(t, libs, "PrimarySync") + assert.Contains(t, libs, "SharedSync-live") + assert.GreaterOrEqual(t, changesDatabaseCalls.Load(), int32(2), "in-memory libraries must still rediscover zones") +} + +func TestGetLibraries_KeepsCachedSharedZonesOnTransientSharedRediscoveryFailure(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + + ps := newHTTPTestPhotosService(t, "library-shared-preserve", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/database/1/com.apple.photos.cloud/production/private/changes/database": + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + case "/database/1/com.apple.photos.cloud/production/shared/changes/database": + http.Error(w, "transient shared discovery failure", http.StatusInternalServerError) + case "/database/1/com.apple.photos.cloud/production/shared/records/query": + http.Error(w, "transient per-zone probe failure", http.StatusInternalServerError) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + }) + + saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared}}) + + libs, err := ps.GetLibraries(ctx) + require.NoError(t, err) + assert.Contains(t, libs, "PrimarySync") + assert.Contains(t, libs, "SharedSync-cached") + + data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json")) + require.NoError(t, err) + assert.Contains(t, string(data), "SharedSync-cached") +} + +func TestGetLibraries_DropsCachedSharedZoneOnZoneNotFoundProbe(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + + ps := newHTTPTestPhotosService(t, "library-shared-drop", func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/database/1/com.apple.photos.cloud/production/private/changes/database": + writeJSON(t, w, map[string]any{"zones": []map[string]any{{"zoneID": map[string]any{"zoneName": "PrimarySync", "ownerRecordName": "owner", "zoneType": "REGULAR_CUSTOM_ZONE"}}}}) + case "/database/1/com.apple.photos.cloud/production/shared/changes/database": + http.Error(w, "transient shared discovery failure", http.StatusInternalServerError) + case "/database/1/com.apple.photos.cloud/production/shared/records/query": + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{ + "serverErrorCode": "ZONE_NOT_FOUND", + "reason": "Zone does not exist", + "errorClass": "ZONE_NOT_FOUND", + "error": "ZONE_NOT_FOUND" + }`)) + default: + http.Error(w, "unexpected request", http.StatusInternalServerError) + } + }) + + saveJSONCache(ps.client.CacheDir(), "libraries.json", []cachedLibraryEntry{{ZoneName: "PrimarySync", Area: areaPrivate}, {ZoneName: "SharedSync-cached", Area: areaShared, OwnerRecordName: "owner", ZoneType: "REGULAR_CUSTOM_ZONE"}}) + + libs, err := ps.GetLibraries(ctx) + require.NoError(t, err) + assert.Contains(t, libs, "PrimarySync") + assert.NotContains(t, libs, "SharedSync-cached") + + data, err := os.ReadFile(filepath.Join(ps.client.CacheDir(), "libraries.json")) + require.NoError(t, err) + assert.NotContains(t, string(data), "SharedSync-cached") +} + +func TestGetAlbums_RetriesAfterTransientUserAlbumQueryFailure(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + var topQueries atomic.Int32 + + ps := newHTTPTestPhotosService(t, "albums-retry-top", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" { + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + var body struct { + Query struct { + FilterBy []struct { + FieldName string `json:"fieldName"` + Comparator string `json:"comparator"` + FieldValue struct { + Value string `json:"value"` + } `json:"fieldValue"` + } `json:"filterBy"` + } `json:"query"` + } + if err := readJSONBody(r, &body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(body.Query.FilterBy) != 0 { + http.Error(w, "unexpected child query", http.StatusInternalServerError) + return + } + if topQueries.Add(1) == 1 { + http.Error(w, "transient top-level album failure", http.StatusInternalServerError) + return + } + writeJSON(t, w, map[string]any{ + "records": []map[string]any{testAlbumRecordJSON("User Album", "user-record", 0, "")}, + }) + }) + + lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}} + _, err := lib.GetAlbums(ctx) + assert.Error(t, err, "private library failure should surface as error") + albums, err := lib.GetAlbums(ctx) + require.NoError(t, err) + assert.Contains(t, albums, "User Album") + assert.GreaterOrEqual(t, topQueries.Load(), int32(2), "second GetAlbums call must retry transient top-level failures") +} + +func TestGetAlbums_SharedSyncQueryFailureReturnsSmartAlbumsOnly(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + + ps := newHTTPTestPhotosService(t, "albums-shared-fallback", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{ + "serverErrorCode": "BAD_REQUEST", + "reason": "Index has invalid data", + "errorClass": "BAD_REQUEST", + "error": "BAD_REQUEST" + }`)) + }) + + lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}} + albums, err := lib.GetAlbums(ctx) + require.NoError(t, err) + assert.Contains(t, albums, "All Photos") + assert.NotContains(t, albums, "User Album") + assert.Empty(t, lib.albums, "SharedSync fallback should not cache partial album results") +} + +func TestGetAlbums_SharedSyncUnexpectedFailureReturnsError(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + + ps := newHTTPTestPhotosService(t, "albums-shared-error", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unexpected shared failure", http.StatusInternalServerError) + }) + + lib := &Library{service: ps, zoneID: "SharedSync-test", area: areaPrivate, albums: map[string]*Album{}} + _, err := lib.GetAlbums(ctx) + assert.Error(t, err) + assert.Empty(t, lib.albums, "SharedSync unexpected failures must not cache partial album results") +} + +func TestGetAlbums_RetriesAfterTransientFolderChildQueryFailure(t *testing.T) { + setTestCacheDir(t) + ctx := context.Background() + var childQueries atomic.Int32 + + ps := newHTTPTestPhotosService(t, "albums-retry-folder", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/database/1/com.apple.photos.cloud/production/private/records/query" { + http.Error(w, "unexpected request", http.StatusInternalServerError) + return + } + var body struct { + Query struct { + FilterBy []struct { + FieldName string `json:"fieldName"` + Comparator string `json:"comparator"` + FieldValue struct { + Value string `json:"value"` + } `json:"fieldValue"` + } `json:"filterBy"` + } `json:"query"` + } + if err := readJSONBody(r, &body); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + parentID := "" + for _, filter := range body.Query.FilterBy { + if filter.FieldName == "parentId" { + parentID = filter.FieldValue.Value + } + } + switch parentID { + case "": + writeJSON(t, w, map[string]any{ + "records": []map[string]any{testAlbumRecordJSON("Folder", "folder-record", albumTypeFolder, "")}, + }) + case "folder-record": + if childQueries.Add(1) == 1 { + http.Error(w, "transient child query failure", http.StatusInternalServerError) + return + } + writeJSON(t, w, map[string]any{ + "records": []map[string]any{testAlbumRecordJSON("Child", "child-record", 0, "folder-record")}, + }) + default: + http.Error(w, "unexpected parentId", http.StatusInternalServerError) + } + }) + + lib := &Library{service: ps, zoneID: "PrimarySync", area: areaPrivate, albums: map[string]*Album{}} + _, _ = lib.GetAlbums(ctx) + albums, err := lib.GetAlbums(ctx) + require.NoError(t, err) + folder, ok := albums["Folder"] + require.True(t, ok) + require.True(t, folder.IsFolder) + assert.Contains(t, folder.Children, "Child") + assert.GreaterOrEqual(t, childQueries.Load(), int32(2), "second GetAlbums call must retry transient child query failures") +} + +func TestBuildPhotos(t *testing.T) { + filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG")) + + master := &photoRecord{ + RecordName: "master123", + RecordType: "CPLMaster", + } + master.Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"} + master.Fields.ResOriginalRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 1024, DownloadURL: "https://example.com/photo"}} + master.Fields.ResOriginalWidth = &ckIntField{Value: 4032} + master.Fields.ResOriginalHeight = &ckIntField{Value: 3024} + + asset := &photoRecord{} + asset.Fields.AssetDate = &ckTimestampField{Value: 1700000000000} + asset.Fields.AddedDate = &ckTimestampField{Value: 1700000001000} + + t.Run("basic photo", func(t *testing.T) { + photos := buildPhotos(master, asset) + require.Len(t, photos, 1) + p := photos[0] + assert.Equal(t, "master123", p.ID) + assert.Equal(t, "IMG_0001.JPG", p.Filename) + assert.Equal(t, int64(1024), p.Size) + assert.Equal(t, 4032, p.Width) + assert.Equal(t, 3024, p.Height) + assert.Equal(t, int64(1700000000000), p.AssetDate) + assert.Equal(t, int64(1700000001000), p.AddedDate) + assert.Equal(t, "resOriginalRes", p.ResourceKey) + }) + + t.Run("live photo with MOV companion", func(t *testing.T) { + liveMaster := *master // copy to avoid mutating shared fixture + liveMaster.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 2048, DownloadURL: "https://example.com/video"}} + + photos := buildPhotos(&liveMaster, asset) + require.Len(t, photos, 2) + + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "resOriginalRes", photos[0].ResourceKey) + + assert.Equal(t, "IMG_0001.MOV", photos[1].Filename) + assert.Equal(t, "resOriginalVidComplRes", photos[1].ResourceKey) + assert.Equal(t, int64(2048), photos[1].Size) + assert.Equal(t, "master123", photos[1].ID) + }) + + t.Run("no download URL returns nil", func(t *testing.T) { + noURL := &photoRecord{RecordName: "no-url"} + noURL.Fields.FilenameEnc = master.Fields.FilenameEnc + photos := buildPhotos(noURL, nil) + assert.Nil(t, photos) + }) + + t.Run("no filename returns nil", func(t *testing.T) { + noName := &photoRecord{RecordName: "no-name"} + noName.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(noName, nil) + assert.Nil(t, photos) + }) + + t.Run("nil asset is handled", func(t *testing.T) { + photos := buildPhotos(master, nil) + require.Len(t, photos, 1) + assert.Equal(t, int64(0), photos[0].AssetDate) + }) + + t.Run("invalid base64 filename returns nil", func(t *testing.T) { + bad := &photoRecord{RecordName: "bad-b64"} + bad.Fields.FilenameEnc = &ckStringField{Value: "!!!not-base64!!!", Type: "ENCRYPTED_BYTES"} + bad.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(bad, nil) + assert.Nil(t, photos) + }) + + t.Run("STRING type filenameEnc used as-is", func(t *testing.T) { + rec := &photoRecord{RecordName: "str-name"} + rec.Fields.FilenameEnc = &ckStringField{Value: "plain_photo.heic", Type: "STRING"} + rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(rec, nil) + require.Len(t, photos, 1) + assert.Equal(t, "plain_photo.heic", photos[0].Filename) + }) + + t.Run("NFD filename normalized to NFC", func(t *testing.T) { + // "Café.heic" in NFD: e + combining acute accent (U+0301) + nfd := "Cafe\u0301.heic" + nfc := "Caf\u00e9.heic" + + rec := &photoRecord{RecordName: "nfd-enc"} + rec.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte(nfd)), Type: "ENCRYPTED_BYTES"} + rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(rec, nil) + require.Len(t, photos, 1) + assert.Equal(t, nfc, photos[0].Filename) + + rec2 := &photoRecord{RecordName: "nfd-str"} + rec2.Fields.FilenameEnc = &ckStringField{Value: nfd, Type: "STRING"} + rec2.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos2 := buildPhotos(rec2, nil) + require.Len(t, photos2, 1) + assert.Equal(t, nfc, photos2[0].Filename) + }) + + t.Run("itemType fallback when filenameEnc missing", func(t *testing.T) { + rec := &photoRecord{RecordName: "AaBbCcDd1234"} + rec.Fields.ItemType = &ckStringField{Value: "public.heic"} + rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(rec, nil) + require.Len(t, photos, 1) + assert.Equal(t, "AaBbCcDd1234.heic", photos[0].Filename) + }) + + t.Run("unknown itemType no fallback returns nil", func(t *testing.T) { + rec := &photoRecord{RecordName: "unknown-uti"} + rec.Fields.ItemType = &ckStringField{Value: "com.unknown.weird-format"} + rec.Fields.ResOriginalRes = master.Fields.ResOriginalRes + photos := buildPhotos(rec, nil) + assert.Nil(t, photos) + }) + + t.Run("edited photo version from adjustmentType", func(t *testing.T) { + editAsset := &photoRecord{RecordName: "asset-edited-123"} + editAsset.Fields.AssetDate = asset.Fields.AssetDate + editAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"} + editAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 512, DownloadURL: "https://example.com/edited"}} + editAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"} + + photos := buildPhotos(master, editAsset) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "resOriginalRes", photos[0].ResourceKey) + assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename) + assert.Equal(t, "resJPEGFullRes", photos[1].ResourceKey) + assert.Equal(t, int64(512), photos[1].Size) + assert.Equal(t, "asset-edited-123", photos[1].ID) + }) + + t.Run("slo-mo skips edited version", func(t *testing.T) { + slomoAsset := &photoRecord{RecordName: "asset-slomo"} + slomoAsset.Fields.AssetDate = asset.Fields.AssetDate + slomoAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.video.slomo"} + photos := buildPhotos(master, slomoAsset) + require.Len(t, photos, 1) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + }) + + t.Run("RAW alternative from resOriginalAltRes", func(t *testing.T) { + rawMaster := &photoRecord{RecordName: "master-raw"} + rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc + rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes + rawMaster.Fields.ResOriginalWidth = master.Fields.ResOriginalWidth + rawMaster.Fields.ResOriginalHeight = master.Fields.ResOriginalHeight + rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 8192, DownloadURL: "https://example.com/raw"}} + rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.canon.cr2-raw-image"} + + photos := buildPhotos(rawMaster, nil) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "resOriginalRes", photos[0].ResourceKey) + assert.Equal(t, "IMG_0001.cr2", photos[1].Filename) + assert.Equal(t, "resOriginalAltRes", photos[1].ResourceKey) + assert.Equal(t, int64(8192), photos[1].Size) + assert.Equal(t, 4032, photos[1].Width, "RAW alt must inherit width from original") + assert.Equal(t, 3024, photos[1].Height, "RAW alt must inherit height from original") + }) + + t.Run("RAW alt same extension gets -alt suffix", func(t *testing.T) { + dupMaster := &photoRecord{RecordName: "master-dup-ext"} + dupMaster.Fields.FilenameEnc = master.Fields.FilenameEnc + dupMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes + dupMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 4096, DownloadURL: "https://example.com/alt"}} + dupMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "public.jpeg"} + + photos := buildPhotos(dupMaster, nil) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + // Same extension would collide, so -alt suffix added + assert.Equal(t, "IMG_0001-alt.jpg", photos[1].Filename) + }) + + t.Run("edited video via resVidFullRes", func(t *testing.T) { + vidAsset := &photoRecord{RecordName: "asset-vid-edit"} + vidAsset.Fields.AssetDate = asset.Fields.AssetDate + vidAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"} + vidAsset.Fields.ResVidFullRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 2048, DownloadURL: "https://example.com/vid-edit"}} + vidAsset.Fields.ResVidFullFileType = &ckStringField{Value: "public.mpeg-4"} + + photos := buildPhotos(master, vidAsset) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "IMG_0001-edited.mp4", photos[1].Filename) + assert.Equal(t, "resVidFullRes", photos[1].ResourceKey) + }) + + t.Run("edited + RAW on same photo", func(t *testing.T) { + rawMaster := &photoRecord{RecordName: "master-combo"} + rawMaster.Fields.FilenameEnc = master.Fields.FilenameEnc + rawMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes + rawMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 8192, DownloadURL: "https://example.com/raw"}} + rawMaster.Fields.ResOriginalAltFileType = &ckStringField{Value: "com.nikon.raw-image"} + + comboAsset := &photoRecord{RecordName: "asset-combo"} + comboAsset.Fields.AssetDate = asset.Fields.AssetDate + comboAsset.Fields.AdjustmentType = &ckStringField{Value: "com.apple.photos"} + comboAsset.Fields.ResJPEGFullRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 512, DownloadURL: "https://example.com/edited"}} + comboAsset.Fields.ResJPEGFullFileType = &ckStringField{Value: "public.jpeg"} + + photos := buildPhotos(rawMaster, comboAsset) + require.Len(t, photos, 3) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "IMG_0001-edited.jpg", photos[1].Filename) + assert.Equal(t, "IMG_0001.nef", photos[2].Filename) + }) + + t.Run("RAW alt with nil file type uses original extension", func(t *testing.T) { + noTypeMaster := &photoRecord{RecordName: "master-notype"} + noTypeMaster.Fields.FilenameEnc = master.Fields.FilenameEnc + noTypeMaster.Fields.ResOriginalRes = master.Fields.ResOriginalRes + noTypeMaster.Fields.ResOriginalAltRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 4096, DownloadURL: "https://example.com/alt-notype"}} + // No ResOriginalAltFileType set + + photos := buildPhotos(noTypeMaster, nil) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + // Same extension collision triggers -alt suffix + assert.Equal(t, "IMG_0001-alt.JPG", photos[1].Filename) + }) + + t.Run("extensionless filename gets .MOV companion", func(t *testing.T) { + noExt := &photoRecord{RecordName: "no-ext"} + noExt.Fields.FilenameEnc = &ckStringField{Value: base64.StdEncoding.EncodeToString([]byte("IMG_NOEXT")), Type: "ENCRYPTED_BYTES"} + noExt.Fields.ResOriginalRes = master.Fields.ResOriginalRes + noExt.Fields.ResOriginalVidComplRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 512, DownloadURL: "https://example.com/v"}} + photos := buildPhotos(noExt, nil) + require.Len(t, photos, 2) + assert.Equal(t, "IMG_NOEXT", photos[0].Filename) + assert.Equal(t, "IMG_NOEXT.MOV", photos[1].Filename) + }) +} + +func TestParsePhotoRecords(t *testing.T) { + filename := base64.StdEncoding.EncodeToString([]byte("IMG_0001.JPG")) + + records := []photoRecord{ + {RecordName: "master1", RecordType: "CPLMaster"}, + {RecordName: "asset1", RecordType: "CPLAsset"}, + {RecordName: "master2", RecordType: "CPLMaster"}, + {RecordName: "asset2", RecordType: "CPLAsset"}, + } + // Set up master fields + for i := range records { + if records[i].RecordType == "CPLMaster" { + records[i].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"} + records[i].Fields.ResOriginalRes = &ckResourceField{Value: struct { + Size int64 `json:"size"` + DownloadURL string `json:"downloadURL"` + }{Size: 1024, DownloadURL: "https://example.com/dl"}} + } + } + // Link assets to masters + records[1].Fields.MasterRef = &ckReferenceField{Value: struct { + RecordName string `json:"recordName"` + }{RecordName: "master1"}} + records[3].Fields.MasterRef = &ckReferenceField{Value: struct { + RecordName string `json:"recordName"` + }{RecordName: "master2"}} + + photos := parsePhotoRecords(records) + require.Len(t, photos, 2) + assert.Equal(t, "master1", photos[0].ID) + assert.Equal(t, "master2", photos[1].ID) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + + t.Run("orphan master without asset", func(t *testing.T) { + orphan := []photoRecord{ + {RecordName: "orphan-master", RecordType: "CPLMaster"}, + } + orphan[0].Fields.FilenameEnc = &ckStringField{Value: filename, Type: "ENCRYPTED_BYTES"} + orphan[0].Fields.ResOriginalRes = records[0].Fields.ResOriginalRes + photos := parsePhotoRecords(orphan) + require.Len(t, photos, 1) + assert.Equal(t, "orphan-master", photos[0].ID) + }) + + t.Run("empty records", func(t *testing.T) { + photos := parsePhotoRecords(nil) + assert.Nil(t, photos) + }) +} + +func TestClassifySmartAlbums(t *testing.T) { + makeMaster := func(fileType string) *photoRecord { + m := &photoRecord{} + if fileType != "" { + m.Fields.ResOriginalFileType = &ckStringField{Value: fileType} + } + return m + } + makeAsset := func(subtype, subtypeV2, favorite, hidden int) *photoRecord { + a := &photoRecord{} + a.Fields.AssetSubtype = &ckIntField{Value: subtype} + a.Fields.AssetSubtypeV2 = &ckIntField{Value: subtypeV2} + a.Fields.IsFavorite = &ckIntField{Value: favorite} + a.Fields.IsHidden = &ckIntField{Value: hidden} + return a + } + + tests := []struct { + name string + master *photoRecord + asset *photoRecord + expected []string + }{ + {"regular photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 0), []string{"All Photos"}}, + {"video mpeg4", makeMaster("public.mpeg-4"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}}, + {"video quicktime", makeMaster("com.apple.quicktime-movie"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Videos"}}, + {"favorite photo", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 0), []string{"All Photos", "Favorites"}}, + {"hidden photo", makeMaster("public.jpeg"), makeAsset(0, 0, 0, 1), []string{"Hidden"}}, + {"screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 0, 0), []string{"All Photos", "Screenshots"}}, + {"favorited screenshot", makeMaster("public.jpeg"), makeAsset(0, 3, 1, 0), []string{"All Photos", "Favorites", "Screenshots"}}, + {"slo-mo", makeMaster("public.mpeg-4"), makeAsset(100, 0, 0, 0), []string{"All Photos", "Slo-mo"}}, + {"time-lapse", makeMaster("public.mpeg-4"), makeAsset(101, 0, 0, 0), []string{"All Photos", "Time-lapse"}}, + {"panorama", makeMaster("public.jpeg"), makeAsset(1, 0, 0, 0), []string{"All Photos", "Panoramas"}}, + {"live photo", makeMaster("public.jpeg"), makeAsset(0, 2, 0, 0), []string{"All Photos", "Live"}}, + {"nil asset", makeMaster("public.jpeg"), nil, []string{"All Photos"}}, + {"no file type", makeMaster(""), makeAsset(0, 0, 0, 0), []string{"All Photos"}}, + {"hidden favorite", makeMaster("public.jpeg"), makeAsset(0, 0, 1, 1), []string{"Hidden", "Favorites"}}, + {"burst photo", makeMaster("public.jpeg"), func() *photoRecord { + a := makeAsset(0, 0, 0, 0) + a.Fields.BurstID = &ckStringField{Value: "B7A3F2E1-4D5C-6789-ABCD-EF0123456789"} + return a + }(), []string{"All Photos", "Bursts"}}, + {"portrait photo", makeMaster("public.jpeg"), func() *photoRecord { + a := makeAsset(0, 0, 0, 0) + a.Fields.AdjustmentRenderType = &ckIntField{Value: 2} // PORTRAIT bit + return a + }(), []string{"All Photos", "Portrait"}}, + {"long exposure", makeMaster("public.jpeg"), func() *photoRecord { + a := makeAsset(0, 0, 0, 0) + a.Fields.AdjustmentRenderType = &ckIntField{Value: 4} // LONG_EXPOSURE bit + return a + }(), []string{"All Photos", "Long Exposure"}}, + {"portrait + long exposure", makeMaster("public.jpeg"), func() *photoRecord { + a := makeAsset(0, 0, 0, 0) + a.Fields.AdjustmentRenderType = &ckIntField{Value: 6} // both bits + return a + }(), []string{"All Photos", "Portrait", "Long Exposure"}}, + {"animated gif", makeMaster("com.compuserve.gif"), makeAsset(0, 0, 0, 0), []string{"All Photos", "Animated"}}, + {"soft-deleted photo", makeMaster("public.jpeg"), func() *photoRecord { + a := makeAsset(0, 0, 1, 0) // favorite=1 should be ignored when deleted + a.Fields.IsDeleted = &ckIntField{Value: 1} + return a + }(), []string{"Recently Deleted"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := classifySmartAlbums(tt.master, tt.asset) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestGetPhotoByName_CacheHit(t *testing.T) { + album := &Album{ + Name: "Videos", + ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video", + } + album.photoCache = map[string]*Photo{ + "IMG_0001.JPG": {ID: "m1", Filename: "IMG_0001.JPG", Size: 1024}, + "IMG_0002.MOV": {ID: "m2", Filename: "IMG_0002.MOV", Size: 2048}, + } + + t.Run("found in cache", func(t *testing.T) { + photo, err := album.GetPhotoByName(context.Background(), "IMG_0001.JPG") + require.NoError(t, err) + assert.Equal(t, "m1", photo.ID) + assert.Equal(t, int64(1024), photo.Size) + }) + + t.Run("not in cache", func(t *testing.T) { + // GetPhotoByName with a populated cache but missing filename + // should return error (cache exists but name not found) + _, err := album.GetPhotoByName(context.Background(), "MISSING.JPG") + assert.Error(t, err) + assert.Contains(t, err.Error(), "MISSING.JPG") + }) +} + +func TestDeduplicateFilenames(t *testing.T) { + t.Run("no duplicates unchanged", func(t *testing.T) { + photos := []*Photo{ + {ID: "master1", Filename: "IMG_0001.JPG"}, + {ID: "master2", Filename: "IMG_0002.JPG"}, + } + deduplicateFilenames(photos) + assert.Equal(t, "IMG_0001.JPG", photos[0].Filename) + assert.Equal(t, "IMG_0002.JPG", photos[1].Filename) + }) + + t.Run("all duplicates get full ID suffix", func(t *testing.T) { + photos := []*Photo{ + {ID: "AQJ2Fq0Px7pGM", Filename: "camphoto_001.jpg"}, + {ID: "BRK3Gr1Qy8qHN", Filename: "camphoto_001.jpg"}, + {ID: "CSL4Hs2Rz9rIO", Filename: "camphoto_001.jpg"}, + } + deduplicateFilenames(photos) + assert.Equal(t, "camphoto_001_AQJ2Fq0Px7pGM.jpg", photos[0].Filename) + assert.Equal(t, "camphoto_001_BRK3Gr1Qy8qHN.jpg", photos[1].Filename) + assert.Equal(t, "camphoto_001_CSL4Hs2Rz9rIO.jpg", photos[2].Filename) + }) + + t.Run("no extension handled", func(t *testing.T) { + photos := []*Photo{ + {ID: "masterA", Filename: "noext"}, + {ID: "masterB", Filename: "noext"}, + } + deduplicateFilenames(photos) + assert.Equal(t, "noext_masterA", photos[0].Filename) + assert.Equal(t, "noext_masterB", photos[1].Filename) + }) + + t.Run("deterministic across runs", func(t *testing.T) { + mk := func() []*Photo { + return []*Photo{ + {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"}, + {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"}, + } + } + p1 := mk() + p2 := mk() + deduplicateFilenames(p1) + deduplicateFilenames(p2) + assert.Equal(t, p1[0].Filename, p2[0].Filename) + assert.Equal(t, p1[1].Filename, p2[1].Filename) + }) + + t.Run("stable when new duplicate added", func(t *testing.T) { + // Before: two duplicates + before := []*Photo{ + {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"}, + {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"}, + } + deduplicateFilenames(before) + nameA := before[0].Filename + nameB := before[1].Filename + + // After: third duplicate added - existing names must not change + after := []*Photo{ + {ID: "AQJ2Fq0Px7pGM", Filename: "dup.jpg"}, + {ID: "BRK3Gr1Qy8qHN", Filename: "dup.jpg"}, + {ID: "CSL4Hs2Rz9rIO", Filename: "dup.jpg"}, + } + deduplicateFilenames(after) + assert.Equal(t, nameA, after[0].Filename, "existing file A must keep same name") + assert.Equal(t, nameB, after[1].Filename, "existing file B must keep same name") + }) +} + +func TestDeduplicateFilenamesSharedPointers(t *testing.T) { + // Photos classified into multiple smart albums share *Photo pointers + // deduplicateFilenames mutates Filename in place, so shared pointers + // across albums get cross-contaminated unless deep-copied first + shared := &Photo{ID: "master1", Filename: "dup.jpg"} + other := &Photo{ID: "master2", Filename: "dup.jpg"} + + albumA := []*Photo{shared, other} + albumB := []*Photo{shared} // same pointer, unique in this album + + // Simulate the fix: deep-copy before dedup (same as applyPendingDelta) + dedupedA := make([]*Photo, len(albumA)) + for i, p := range albumA { + cp := *p + dedupedA[i] = &cp + } + deduplicateFilenames(dedupedA) + + // albumA's dedup copies got suffixes + assert.Contains(t, dedupedA[0].Filename, "_master1") + assert.Contains(t, dedupedA[1].Filename, "_master2") + + // Original shared pointer is untouched - albumB sees clean filename + assert.Equal(t, "dup.jpg", albumB[0].Filename, "deep copy must prevent cross-album contamination") + assert.Equal(t, "dup.jpg", shared.Filename, "original pointer must be unmodified") +} + +func TestDeltaContainsAlbumChanges(t *testing.T) { + t.Run("CPLAlbum record triggers invalidation", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`), + json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`), + json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`), + } + assert.True(t, deltaContainsAlbumChanges(records)) + }) + + t.Run("no CPLAlbum does not trigger", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster"}`), + json.RawMessage(`{"recordName":"asset1","recordType":"CPLAsset"}`), + json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation"}`), + } + assert.False(t, deltaContainsAlbumChanges(records)) + }) + + t.Run("empty records does not trigger", func(t *testing.T) { + assert.False(t, deltaContainsAlbumChanges(nil)) + }) + + t.Run("malformed JSON skipped", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{invalid`), + json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`), + } + assert.True(t, deltaContainsAlbumChanges(records)) + }) +} + +func TestParseDeltaRecords(t *testing.T) { + t.Run("deleted CPLAsset marks both asset and master IDs", func(t *testing.T) { + // Edited entries use asset.RecordName as ID; ghost entries survive + // if only masterID is in deletedIDs + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"master-AAA","recordType":"CPLMaster","deleted":true}`), + json.RawMessage(`{"recordName":"asset-BBB","recordType":"CPLAsset","deleted":true,"fields":{"masterRef":{"value":{"recordName":"master-AAA"}}}}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.deletedIDs["master-AAA"], "master ID from CPLMaster deletion") + assert.True(t, r.deletedIDs["asset-BBB"], "asset ID from CPLAsset deletion (for -edited entries)") + }) + + t.Run("deleted CPLAsset without masterRef still tracks asset ID", func(t *testing.T) { + // CloudKit may strip fields from deleted records + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"asset-CCC","recordType":"CPLAsset","deleted":true}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.deletedIDs["asset-CCC"], "asset ID tracked even without masterRef") + assert.Equal(t, 1, len(r.deletedIDs), "only asset ID, no master") + }) + + t.Run("deleted CPLContainerRelation extracts album from recordName", func(t *testing.T) { + // Deleted relations lack fields; album parsed from "assetID-IN-albumRecordName" + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-456","recordType":"CPLContainerRelation","deleted":true}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.changedAlbumRecords["album-uuid-456"], "album extracted from deleted relation recordName") + }) + + t.Run("deleted relation without recordType still extracts album from recordName", func(t *testing.T) { + // changes/zone omits recordType for deleted CPLContainerRelation records + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-654","deleted":true}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.changedAlbumRecords["album-uuid-654"], "album extracted from deleted relation with missing recordType") + }) + + t.Run("non-deleted CPLContainerRelation uses containerId STRING field", func(t *testing.T) { + // CloudKit relation records expose containerId as the canonical album field + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"rel1","recordType":"CPLContainerRelation","fields":{"containerId":{"value":"album-uuid-789"}}}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.changedAlbumRecords["album-uuid-789"], "album extracted from containerId STRING field") + }) + + t.Run("non-deleted CPLContainerRelation falls back to recordName when fields missing", func(t *testing.T) { + // changes/zone can return live relation records with empty fields + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"assetXYZ-IN-album-uuid-987","recordType":"CPLContainerRelation","fields":{},"deleted":false}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.changedAlbumRecords["album-uuid-987"], "album extracted from live relation recordName fallback") + }) + + t.Run("deleted relation with unexpected recordName format is safe", func(t *testing.T) { + // No "-IN-" in recordName - must not panic or add garbage + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"weirdformat","recordType":"CPLContainerRelation","deleted":true}`), + } + r := parseDeltaRecords(records) + assert.Empty(t, r.changedAlbumRecords, "no album invalidated for malformed recordName") + }) + + t.Run("CPLAlbum sets albumMetadataChanged", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"album1","recordType":"CPLAlbum"}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.albumMetadataChanged) + }) + + t.Run("asset-only update detected", func(t *testing.T) { + // CPLAsset changed but no CPLMaster in delta - triggers smart album invalidation + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"asset-DDD","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-EEE"}}}}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.hasAssetOnlyUpdates, "asset without matching master = asset-only update") + }) + + t.Run("asset with matching master is not asset-only", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"master-FFF","recordType":"CPLMaster","fields":{"resOriginalRes":{"value":{"downloadURL":"https://example.com"}}}}`), + json.RawMessage(`{"recordName":"asset-GGG","recordType":"CPLAsset","fields":{"masterRef":{"value":{"recordName":"master-FFF"}}}}`), + } + r := parseDeltaRecords(records) + assert.False(t, r.hasAssetOnlyUpdates) + assert.Equal(t, 1, len(r.newMasters)) + assert.Equal(t, 1, len(r.newAssets)) + }) + + t.Run("unknown record types are silently skipped", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{"recordName":"x","recordType":"CPLSomethingNew"}`), + } + r := parseDeltaRecords(records) + assert.Empty(t, r.deletedIDs) + assert.Empty(t, r.newMasters) + assert.Empty(t, r.newAssets) + assert.Empty(t, r.changedAlbumRecords) + assert.False(t, r.albumMetadataChanged) + }) + + t.Run("malformed JSON records skipped without panic", func(t *testing.T) { + records := []json.RawMessage{ + json.RawMessage(`{invalid`), + json.RawMessage(`{"recordName":"master1","recordType":"CPLMaster","deleted":true}`), + } + r := parseDeltaRecords(records) + assert.True(t, r.deletedIDs["master1"], "valid record after malformed one still parsed") + }) + + t.Run("empty records produces empty result", func(t *testing.T) { + r := parseDeltaRecords(nil) + assert.Empty(t, r.deletedIDs) + assert.False(t, r.albumMetadataChanged) + assert.False(t, r.hasAssetOnlyUpdates) + }) +} + +func TestApplyPendingDelta_EmptyAlbumsClearsPending(t *testing.T) { + // When checkForChanges eagerly clears lib.albums (CPLAlbum delta), + // applyPendingDelta must clear pendingDelta instead of leaving it stuck + // Without this fix, pendingDelta stays non-nil forever and all future + // checkForChanges calls skip (line 912: pendingDelta != nil) + lib := &Library{ + albums: make(map[string]*Album), // empty - simulates eager clear + pendingDelta: &deltaPayload{ + records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)}, + syncToken: "old-token", + }, + } + result := lib.applyPendingDelta(context.Background()) + assert.False(t, result, "must return false when albums empty") + assert.Nil(t, lib.pendingDelta, "must clear pendingDelta to avoid permanent stuck state") +} + +func TestApplyPendingDelta_InvalidatesNestedAlbum(t *testing.T) { + oldCacheDir := config.GetCacheDir() + t.Cleanup(func() { + _ = config.SetCacheDir(oldCacheDir) + }) + require.NoError(t, config.SetCacheDir(t.TempDir())) + + ps := &PhotosService{client: &Client{remoteName: "delta-nested-test"}} + lib := &Library{ + service: ps, + zoneID: "PrimarySync", + area: areaPrivate, + albums: make(map[string]*Album), + } + child := lib.newUserAlbum("Child", "child-record") + child.SetTestPhotoCache(map[string]*Photo{ + "one.jpg": {ID: "master1", Filename: "one.jpg"}, + }) + child.saveDiskCache([]*Photo{{ID: "master1", Filename: "one.jpg"}}) + + folder := &Album{ + Name: "Folder", + RecordName: "folder-record", + lib: lib, + IsFolder: true, + Children: map[string]*Album{ + "Child": child, + }, + } + lib.albums[folder.Name] = folder + lib.pendingDelta = &deltaPayload{ + records: []json.RawMessage{json.RawMessage(`{"recordName":"asset123-IN-child-record","deleted":true}`)}, + syncToken: "next-token", + } + + cacheFile := filepath.Join(lib.zoneCacheDir(), albumCacheKey(child.ObjectType)+".json") + _, err := os.Stat(cacheFile) + require.NoError(t, err, "nested child album cache file should exist before invalidation") + + result := lib.applyPendingDelta(context.Background()) + assert.True(t, result) + assert.Nil(t, lib.pendingDelta) + + child.mu.Lock() + assert.Nil(t, child.photoCache, "nested child album memory cache should be invalidated") + child.mu.Unlock() + _, err = os.Stat(cacheFile) + assert.ErrorIs(t, err, os.ErrNotExist, "nested child album disk cache should be removed") + data, err := os.ReadFile(filepath.Join(lib.zoneCacheDir(), "syncToken")) + require.NoError(t, err) + assert.Equal(t, "next-token", string(data)) +} + +func TestFlushCaches_NoPendingDeltaRace(t *testing.T) { + // Verify FlushCaches and applyPendingDelta don't race on pendingDelta. + // FlushCaches must acquire deltaMu before writing pendingDelta. + ps := &PhotosService{ + client: &Client{remoteName: "race-test"}, + libraries: make(map[string]*Library), + } + lib := &Library{ + service: ps, + zoneID: "PrimarySync", + area: areaPrivate, + albums: make(map[string]*Album), + } + ps.libraries["PrimarySync"] = lib + + // Run concurrent FlushCaches + applyPendingDelta + // The race detector will flag unsynchronized access to pendingDelta + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < 100; i++ { + lib.deltaMu.Lock() + lib.pendingDelta = &deltaPayload{ + records: []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)}, + syncToken: "tok", + } + lib.deltaMu.Unlock() + lib.applyPendingDelta(context.Background()) + } + }() + for i := 0; i < 100; i++ { + ps.FlushCaches() + // Re-add the library since FlushCaches clears it + ps.mu.Lock() + ps.libraries["PrimarySync"] = lib + ps.mu.Unlock() + } + <-done +} + +func TestBuildSmartAlbums(t *testing.T) { + albums := buildSmartAlbums() + + // Must produce exactly 14 smart albums + assert.Equal(t, 14, len(albums), "expected 14 smart albums") + + // All expected names present + expected := []string{ + "All Photos", "Favorites", "Videos", "Screenshots", "Live", + "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure", + "Animated", "Bursts", "Hidden", "Recently Deleted", + } + for _, name := range expected { + a, ok := albums[name] + require.True(t, ok, "missing smart album %q", name) + assert.NotEmpty(t, a.ObjectType, "album %q has no ObjectType", name) + assert.NotEmpty(t, a.ListType, "album %q has no ListType", name) + assert.NotEmpty(t, a.Direction, "album %q has no Direction", name) + } + + // Verify special albums have correct configuration + assert.Equal(t, "DESCENDING", albums["Recently Deleted"].Direction) + assert.Empty(t, albums["All Photos"].Filters, "All Photos should have no filters") + assert.Empty(t, albums["Bursts"].Filters, "Bursts should have no filters") + assert.Empty(t, albums["Hidden"].Filters, "Hidden should have no filters") + + // Verify filter-based albums have exactly one smartAlbum filter + for _, name := range []string{"Favorites", "Videos", "Screenshots", "Live", "Slo-mo", "Time-lapse", "Panoramas", "Portrait", "Long Exposure", "Animated"} { + a := albums[name] + require.Len(t, a.Filters, 1, "album %q should have exactly one filter", name) + assert.Equal(t, "smartAlbum", a.Filters[0].FieldName) + assert.Equal(t, "EQUALS", a.Filters[0].Comparator) + } +} + +func TestAtomicWriteFile(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "test.json") + + // Successful write + err := atomicWriteFile(target, []byte(`{"key":"value"}`)) + require.NoError(t, err) + data, err := os.ReadFile(target) + require.NoError(t, err) + assert.Equal(t, `{"key":"value"}`, string(data)) + + // Overwrite existing + err = atomicWriteFile(target, []byte(`{"key":"updated"}`)) + require.NoError(t, err) + data, err = os.ReadFile(target) + require.NoError(t, err) + assert.Equal(t, `{"key":"updated"}`, string(data)) + + // No .tmp file left behind + _, err = os.Stat(target + ".tmp") + assert.True(t, os.IsNotExist(err), "temp file should not persist") +} + +func TestAlbumCacheKey(t *testing.T) { + k1 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted") + k2 := albumCacheKey("CPLAssetInSmartAlbumByAssetDate:Video") + k3 := albumCacheKey("CPLAssetByAssetDateWithoutHiddenOrDeleted") + + assert.Len(t, k1, 16, "cache key should be 16 hex chars") + assert.Equal(t, k1, k3, "same input should produce same key") + assert.NotEqual(t, k1, k2, "different input should produce different key") + + // Verify filename-safe + for _, c := range k1 { + assert.True(t, (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'), "key should be hex: %c", c) + } +} diff --git a/backend/iclouddrive/api/session.go b/backend/iclouddrive/api/session.go index 004faaa37..89e47a937 100644 --- a/backend/iclouddrive/api/session.go +++ b/backend/iclouddrive/api/session.go @@ -2,16 +2,21 @@ package api import ( "context" + "crypto/sha256" "encoding/base64" + "encoding/hex" + "encoding/json" "fmt" + "io" "maps" "net/http" "net/url" "slices" "strings" + "sync" + "time" "github.com/google/uuid" - "github.com/oracle/oci-go-sdk/v65/common" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/lib/rest" @@ -32,6 +37,7 @@ type Session struct { Cookies []*http.Cookie `json:"cookies"` AccountInfo AccountInfo `json:"account_info"` + mu sync.Mutex `json:"-"` // protects session fields during concurrent Request calls srv *rest.Client `json:"-"` needs2FA bool `json:"-"` // set when SRP signin returns 409 } @@ -45,20 +51,117 @@ type srpInitResponse struct { C string `json:"c"` } -// String returns the session as a string -// func (s *Session) String() string { -// jsession, _ := json.Marshal(s) -// return string(jsession) -// } - -// Request makes a request -func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) { - resp, err := s.srv.CallJSON(ctx, &opts, &request, &response) - - if err != nil { - return resp, err +func cookieValueFingerprint(value string) string { + if value == "" { + return "" } + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:6]) +} +func authStateBodySummary(body []byte) string { + summary := []string{fmt.Sprintf("bytes=%d", len(body)), "hash=" + cookieValueFingerprint(string(body))} + var top map[string]json.RawMessage + if err := json.Unmarshal(body, &top); err != nil { + return strings.Join(summary, " ") + } + keys := make([]string, 0, len(top)) + for key := range top { + keys = append(keys, key) + } + slices.Sort(keys) + summary = append(summary, fmt.Sprintf("keys=%v", keys)) + if _, ok := top["phoneNumberVerification"]; ok { + summary = append(summary, "wrapped=true") + } + return strings.Join(summary, " ") +} + +func cookieDebugSummary(c *http.Cookie) string { + parts := []string{c.Name} + if c.Value == "" { + parts = append(parts, "empty", "len=0") + } else { + parts = append(parts, "set", fmt.Sprintf("len=%d", len(c.Value)), "hash="+cookieValueFingerprint(c.Value)) + } + if c.Path != "" { + parts = append(parts, "path="+c.Path) + } + if c.Domain != "" { + parts = append(parts, "domain="+c.Domain) + } + if c.MaxAge != 0 { + parts = append(parts, fmt.Sprintf("maxAge=%d", c.MaxAge)) + } + if !c.Expires.IsZero() { + parts = append(parts, "expires") + } + return strings.Join(parts, ",") +} + +func cookieDebugSummaries(cookies []*http.Cookie) []string { + if len(cookies) == 0 { + return nil + } + out := make([]string, 0, len(cookies)) + for _, c := range cookies { + out = append(out, cookieDebugSummary(c)) + } + return out +} + +func cookieJarDebugSummaries(cookies []*http.Cookie) []string { + if len(cookies) == 0 { + return nil + } + out := make([]string, 0, len(cookies)) + for _, c := range cookies { + summary := c.Name + if c.Value == "" { + summary += ",empty,len=0" + } else { + summary += fmt.Sprintf(",len=%d,hash=%s", len(c.Value), cookieValueFingerprint(c.Value)) + } + out = append(out, summary) + } + return out +} + +func (s *Session) mergeCookies(cookies []*http.Cookie) { + if len(cookies) == 0 { + return + } + existing := make(map[string]int, len(s.Cookies)) + for i, c := range s.Cookies { + existing[c.Name] = i + } + for _, c := range cookies { + if c.Value == "" { + if idx, ok := existing[c.Name]; ok { + fs.Debugf(nil, "iclouddrive: deleting empty auth cookie: %s", cookieDebugSummary(c)) + s.Cookies = append(s.Cookies[:idx], s.Cookies[idx+1:]...) + delete(existing, c.Name) + for j := idx; j < len(s.Cookies); j++ { + existing[s.Cookies[j].Name] = j + } + } else { + fs.Debugf(nil, "iclouddrive: ignoring empty auth cookie tombstone for missing cookie: %s", cookieDebugSummary(c)) + } + continue + } + if idx, ok := existing[c.Name]; ok { + s.Cookies[idx] = c + } else { + s.Cookies = append(s.Cookies, c) + existing[c.Name] = len(s.Cookies) - 1 + } + } +} + +// extractHeaders reads Apple session headers from an HTTP response into the session +// Does NOT acquire s.mu - caller is responsible for locking if needed +func (s *Session) extractHeaders(resp *http.Response) { + s.mergeCookies(resp.Cookies()) if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" { s.AccountCountry = val } @@ -77,7 +180,17 @@ func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, resp if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" { s.AuthAttributes = val } +} +// Request makes a JSON API request and extracts session headers +func (s *Session) Request(ctx context.Context, opts rest.Opts, request any, response any) (*http.Response, error) { + resp, err := s.srv.CallJSON(ctx, &opts, &request, &response) + if err != nil { + return resp, err + } + s.mu.Lock() + s.extractHeaders(resp) + s.mu.Unlock() return resp, nil } @@ -89,13 +202,8 @@ func (s *Session) Requires2FA() bool { return s.AccountInfo.DsInfo != nil && s.AccountInfo.DsInfo.HsaVersion == 2 && s.AccountInfo.HsaChallengeRequired } -// SignIn performs SRP-based authentication against Apple's idmsa endpoint. +// SignIn performs SRP-based authentication against Apple's idmsa endpoint func (s *Session) SignIn(ctx context.Context, appleID, password string) error { - // Apple's SRP implementation expects a lowercase account name. - // The old plaintext flow didn't need this because the server normalized - // it, but SRP uses the username in client-side proof computation (M1). - appleID = strings.ToLower(appleID) - // Step 1: Initialize the auth session if err := s.authStart(ctx); err != nil { return fmt.Errorf("authStart: %w", err) @@ -106,8 +214,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error { return fmt.Errorf("authFederate: %w", err) } - // Step 3: SRP init — send client public value A, get salt + B - client := newSRPClient() + // Step 3: SRP init - send client public value A, get salt + B + client, err := newSRPClient() + if err != nil { + return fmt.Errorf("newSRPClient: %w", err) + } aBase64 := base64.StdEncoding.EncodeToString(client.getABytes()) initResp, err := s.authSRPInit(ctx, aBase64, appleID) @@ -130,9 +241,11 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error { if err != nil { return fmt.Errorf("derivePassword: %w", err) } - client.processChallenge([]byte(appleID), derivedKey, salt, serverB) + if err := client.processChallenge([]byte(appleID), derivedKey, salt, serverB); err != nil { + return fmt.Errorf("processChallenge: %w", err) + } - // Step 5: Complete — send M1, M2 proofs + // Step 5: Complete - send M1, M2 proofs m1Base64 := base64.StdEncoding.EncodeToString(client.M1) m2Base64 := base64.StdEncoding.EncodeToString(client.M2) @@ -143,7 +256,7 @@ func (s *Session) SignIn(ctx context.Context, appleID, password string) error { return nil } -// authStart initializes the SRP auth session by hitting the authorize/signin endpoint. +// authStart initializes the SRP auth session by hitting the authorize/signin endpoint func (s *Session) authStart(ctx context.Context) error { if s.FrameID == "" { s.FrameID = strings.ToLower(uuid.New().String()) @@ -184,20 +297,11 @@ func (s *Session) authStart(ctx context.Context) error { return fmt.Errorf("authStart: unexpected status %s", resp.Status) } - if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" { - s.AuthAttributes = val - } - if val := resp.Header.Get("scnt"); val != "" { - s.Scnt = val - } - if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" { - s.SessionID = val - } - + s.extractHeaders(resp) return nil } -// authFederate submits the account name to Apple's federate endpoint. +// authFederate submits the account name to Apple's federate endpoint func (s *Session) authFederate(ctx context.Context, accountName string) error { values := map[string]any{ "accountName": accountName, @@ -224,16 +328,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error { } _ = resp.Body.Close() - if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" { - s.AuthAttributes = val - } - if val := resp.Header.Get("scnt"); val != "" { - s.Scnt = val - } - if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" { - s.SessionID = val - } - + s.extractHeaders(resp) if resp.StatusCode != http.StatusOK { return fmt.Errorf("authFederate: unexpected status %s", resp.Status) } @@ -241,7 +336,7 @@ func (s *Session) authFederate(ctx context.Context, accountName string) error { } // authSRPInit sends the client's public value A to the server and retrieves -// the salt, server public value B, iteration count, protocol, and challenge. +// the salt, server public value B, iteration count, protocol, and challenge func (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string) (*srpInitResponse, error) { values := map[string]any{ "a": aBase64, @@ -267,18 +362,12 @@ func (s *Session) authSRPInit(ctx context.Context, aBase64, accountName string) return nil, err } - if val := resp.Header.Get("scnt"); val != "" { - s.Scnt = val - } - if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" { - s.SessionID = val - } - + s.extractHeaders(resp) return &initResp, nil } -// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication. -// Returns nil on success (200 or 409/2FA needed). +// authSRPComplete sends the SRP proofs M1 and M2 to complete authentication +// Returns nil on success (200 or 409/2FA needed) func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2Base64, c string) error { trustTokens := []string{} if s.TrustToken != "" { @@ -305,7 +394,6 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2 ExtraHeaders: s.getSRPAuthHeaders(), RootURL: authEndpoint, IgnoreStatus: true, - NoResponse: true, Body: body, } @@ -313,51 +401,65 @@ func (s *Session) authSRPComplete(ctx context.Context, accountName, m1Base64, m2 if err != nil { return err } + respBody, _ := io.ReadAll(resp.Body) _ = resp.Body.Close() - // Extract updated headers - if val := resp.Header.Get("X-Apple-Auth-Attributes"); val != "" { - s.AuthAttributes = val - } - if val := resp.Header.Get("X-Apple-Session-Token"); val != "" { - s.SessionToken = val - } - if val := resp.Header.Get("scnt"); val != "" { - s.Scnt = val - } - if val := resp.Header.Get("X-Apple-ID-Session-Id"); val != "" { - s.SessionID = val - } - if val := resp.Header.Get("X-Apple-ID-Account-Country"); val != "" { - s.AccountCountry = val - } - if val := resp.Header.Get("X-Apple-TwoSV-Trust-Token"); val != "" { - s.TrustToken = val - } + s.extractHeaders(resp) switch resp.StatusCode { case http.StatusOK: - fs.Debugf("icloud", "SRP sign in successful") + fs.Debugf(nil, "iclouddrive: SRP sign in successful") return nil case http.StatusConflict: - // 409 = 2FA required, this is expected - fs.Debugf("icloud", "SRP sign in requires 2FA") + // 409 = 2FA required + fs.Debugf(nil, "iclouddrive: SRP sign in requires 2FA, response: %s", respBody) s.needs2FA = true return nil + case http.StatusPreconditionFailed: + // 412 = non-2FA account needs repair/complete step + fs.Debugf(nil, "iclouddrive: SRP sign in returned 412, attempting repair/complete") + return s.authRepairComplete(ctx) case http.StatusForbidden: return fmt.Errorf("sign in failed: incorrect username or password") default: - return fmt.Errorf("sign in failed: %s", resp.Status) + return fmt.Errorf("sign in failed: %s: %s", resp.Status, respBody) } } -// getAuthOrigin returns the origin URL for auth requests. -// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpoints. +// authRepairComplete handles the repair flow for non-2FA accounts that return 412 +func (s *Session) authRepairComplete(ctx context.Context) error { + body, err := IntoReader(map[string]any{}) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/repair/complete", + ExtraHeaders: s.getSRPAuthHeaders(), + RootURL: authEndpoint, + IgnoreStatus: true, + NoResponse: true, + Body: body, + } + resp, err := s.srv.Call(ctx, &opts) + if err != nil { + return fmt.Errorf("repair/complete failed: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("repair/complete returned %s", resp.Status) + } + s.extractHeaders(resp) + fs.Debugf(nil, "iclouddrive: repair/complete successful") + return nil +} + +// getAuthOrigin returns the origin URL for auth requests +// Supports both global (idmsa.apple.com) and China (idmsa.apple.com.cn) endpoints func getAuthOrigin() string { return strings.TrimSuffix(authEndpoint, "/appleauth/auth") } -// getSRPAuthHeaders returns headers needed for SRP auth requests. +// getSRPAuthHeaders returns headers needed for SRP auth requests func (s *Session) getSRPAuthHeaders() map[string]string { frameTag := "auth-" + s.FrameID authOrigin := getAuthOrigin() @@ -414,14 +516,110 @@ func (s *Session) AuthWithToken(ctx context.Context) error { } resp, err := s.Request(ctx, opts, nil, &s.AccountInfo) - if err == nil { - s.Cookies = resp.Cookies() + if err != nil { + return err + } + fs.Debugf(nil, "iclouddrive: accountLogin response cookies: %v", cookieDebugSummaries(resp.Cookies())) + fs.Debugf(nil, "iclouddrive: session cookie jar after accountLogin: %v", cookieJarDebugSummaries(s.Cookies)) + + // Acquire PCS cookies if Advanced Data Protection is enabled + if ws := s.AccountInfo.Webservices["ckdatabasews"]; ws != nil && ws.PcsRequired { + fs.Debugf(nil, "iclouddrive: ADP detected (pcsRequired=true)") + if s.hasPCSCookies() { + fs.Debugf(nil, "iclouddrive: PCS cookies already present, skipping acquisition") + } else { + if err := s.acquirePCSCookies(ctx); err != nil { + return err + } + } + } else { + fs.Debugf(nil, "iclouddrive: no ADP (pcsRequired=false)") } + return nil +} + +// hasPCSCookies checks if the required PCS cookies for Photos are already present +func (s *Session) hasPCSCookies() bool { + var hasPhotos, hasSharing bool + for _, c := range s.Cookies { + switch c.Name { + case "X-APPLE-WEBAUTH-PCS-Photos": + hasPhotos = true + case "X-APPLE-WEBAUTH-PCS-Sharing": + hasSharing = true + } + } + return hasPhotos && hasSharing +} + +// acquirePCSCookies requests PCS cookies for ADP-enabled accounts +// May require user approval on a trusted device (polls every 10s, max 5 min) +func (s *Session) acquirePCSCookies(ctx context.Context) error { + fs.Logf(nil, "iclouddrive: Advanced Data Protection enabled, requesting PCS cookies") + const maxAttempts = 30 // 30 * 10s = 5 minutes max + for attempt := 0; attempt < maxAttempts; attempt++ { + fs.Debugf(nil, "iclouddrive: requestPCS outgoing cookies: %v", cookieJarDebugSummaries(s.Cookies)) + values := map[string]any{ + "appName": "photos", + "derivedFromUserAction": true, + } + body, err := IntoReader(values) + if err != nil { + return fmt.Errorf("requestPCS: %w", err) + } + opts := rest.Opts{ + Method: "POST", + Path: "/requestPCS", + ExtraHeaders: s.GetHeaders(map[string]string{}), + RootURL: setupEndpoint, + } + opts.Body = body + var pcsResp struct { + Status string `json:"status"` + Message string `json:"message"` + } + resp, err := s.Request(ctx, opts, nil, &pcsResp) + if err != nil { + return fmt.Errorf("requestPCS: %w", err) + } + fs.Debugf(nil, "iclouddrive: requestPCS response cookies: %v", cookieDebugSummaries(resp.Cookies())) + fs.Debugf(nil, "iclouddrive: requestPCS response: status=%q message=%q cookies=%d", + pcsResp.Status, pcsResp.Message, len(resp.Cookies())) + if pcsResp.Status == "success" { + if !s.hasPCSCookies() { + return fmt.Errorf("requestPCS: server returned success but PCS cookies missing") + } + fs.Logf(nil, "iclouddrive: PCS cookies acquired") + return nil + } + // Device consent required - poll until approved + fs.Logf(nil, "iclouddrive: waiting for device approval for PCS (%s)", pcsResp.Message) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Second): + } + } + return fmt.Errorf("requestPCS: timed out waiting for device approval after 5 minutes") +} + +// RequestPushNotification explicitly requests a push notification to trusted devices +// Required for iOS 26.4+ where the SRP 409 response no longer auto-pushes +func (s *Session) RequestPushNotification(ctx context.Context) error { + opts := rest.Opts{ + Method: "PUT", + Path: "/verify/trusteddevice/securitycode", + ExtraHeaders: s.GetAuthHeaders(map[string]string{}), + RootURL: authEndpoint, + NoResponse: true, + } + + _, err := s.Request(ctx, opts, nil, nil) return err } -// Validate2FACode validates the 2FA code +// Validate2FACode validates the 2FA code from a trusted device push notification func (s *Session) Validate2FACode(ctx context.Context, code string) error { values := map[string]any{"securityCode": map[string]string{"code": code}} body, err := IntoReader(values) @@ -450,6 +648,132 @@ func (s *Session) Validate2FACode(ctx context.Context, code string) error { return fmt.Errorf("validate2FACode failed: %w", err) } +// TrustedPhoneNumber represents a phone number that can receive SMS verification codes +type TrustedPhoneNumber struct { + ID int `json:"id"` + NumberWithDialCode string `json:"numberWithDialCode"` + ObfuscatedNumber string `json:"obfuscatedNumber"` + PushMode string `json:"pushMode"` + NonFTEU bool `json:"nonFTEU"` +} + +// AuthStateResponse is the response from GET /appleauth/auth after sign-in +// Some accounts return fields at top level, others nest them under phoneNumberVerification +type AuthStateResponse struct { + TrustedPhoneNumbers []TrustedPhoneNumber `json:"trustedPhoneNumbers"` + TrustedPhoneNumber *TrustedPhoneNumber `json:"trustedPhoneNumber"` + NoTrustedDevices bool `json:"noTrustedDevices"` + AuthenticationType string `json:"authenticationType"` + Hsa2Account bool `json:"hsa2Account"` + PhoneNumberVerification *AuthStateResponse `json:"phoneNumberVerification"` +} + +// GetAuthState retrieves the current auth state including trusted phone numbers for SMS 2FA +func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error) { + opts := rest.Opts{ + Method: "GET", + Path: "", + ExtraHeaders: s.GetAuthHeaders(map[string]string{}), + RootURL: authEndpoint, + ContentLength: int64Ptr(0), + } + // Use srv.Call directly to capture the raw response body for debugging + resp, err := s.srv.Call(ctx, &opts) + if err != nil { + return nil, fmt.Errorf("getAuthState: %w", err) + } + s.mu.Lock() + s.extractHeaders(resp) + s.mu.Unlock() + body, readErr := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if readErr != nil { + return nil, fmt.Errorf("getAuthState read: %w", readErr) + } + fs.Debugf(nil, "iclouddrive: auth state response summary: %s", authStateBodySummary(body)) + var state AuthStateResponse + if err := json.Unmarshal(body, &state); err != nil { + fs.Debugf(nil, "iclouddrive: auth state parse failed: %s", authStateBodySummary(body)) + return nil, fmt.Errorf("getAuthState unmarshal: %w", err) + } + // Some accounts nest auth data under phoneNumberVerification + if state.PhoneNumberVerification != nil && state.AuthenticationType == "" { + fs.Debugf(nil, "iclouddrive: unwrapping phoneNumberVerification envelope") + n := state.PhoneNumberVerification + state.TrustedPhoneNumbers = n.TrustedPhoneNumbers + state.TrustedPhoneNumber = n.TrustedPhoneNumber + state.NoTrustedDevices = n.NoTrustedDevices + state.AuthenticationType = n.AuthenticationType + state.Hsa2Account = n.Hsa2Account + state.PhoneNumberVerification = nil + } + fs.Debugf(nil, "iclouddrive: auth state: type=%s hsa2=%v noTrustedDevices=%v phones=%d phoneSingular=%v", + state.AuthenticationType, state.Hsa2Account, state.NoTrustedDevices, + len(state.TrustedPhoneNumbers), state.TrustedPhoneNumber != nil) + // Fall back to singular trustedPhoneNumber when plural array is empty + // Some accounts return only the singular form (SMS-only, no trusted devices) + if len(state.TrustedPhoneNumbers) == 0 && state.TrustedPhoneNumber != nil { + fs.Debugf(nil, "iclouddrive: using singular trustedPhoneNumber (id=%d) as fallback", + state.TrustedPhoneNumber.ID) + state.TrustedPhoneNumbers = []TrustedPhoneNumber{*state.TrustedPhoneNumber} + } + return &state, nil +} + +// RequestSMSCode triggers SMS code delivery to a trusted phone number +func (s *Session) RequestSMSCode(ctx context.Context, phoneID int, mode string) error { + values := map[string]any{ + "phoneNumber": map[string]any{"id": phoneID}, + "mode": mode, + } + body, err := IntoReader(values) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "PUT", + Path: "/verify/phone", + ExtraHeaders: s.GetAuthHeaders(map[string]string{}), + RootURL: authEndpoint, + Body: body, + NoResponse: true, + } + _, err = s.Request(ctx, opts, nil, nil) + if err != nil { + return fmt.Errorf("requestSMSCode: %w", err) + } + return nil +} + +// ValidateSMSCode validates a 2FA code received via SMS +func (s *Session) ValidateSMSCode(ctx context.Context, code string, phoneID int, mode string) error { + values := map[string]any{ + "securityCode": map[string]string{"code": code}, + "phoneNumber": map[string]any{"id": phoneID}, + "mode": mode, + } + body, err := IntoReader(values) + if err != nil { + return err + } + opts := rest.Opts{ + Method: "POST", + Path: "/verify/phone/securitycode", + ExtraHeaders: s.GetAuthHeaders(map[string]string{}), + RootURL: authEndpoint, + Body: body, + NoResponse: true, + } + _, err = s.Request(ctx, opts, nil, nil) + if err == nil { + if err := s.TrustSession(ctx); err != nil { + return err + } + return nil + } + return fmt.Errorf("validateSMSCode: %w", err) +} + // TrustSession trusts the session func (s *Session) TrustSession(ctx context.Context) error { opts := rest.Opts{ @@ -458,7 +782,7 @@ func (s *Session) TrustSession(ctx context.Context) error { ExtraHeaders: s.GetAuthHeaders(map[string]string{}), RootURL: authEndpoint, NoResponse: true, - ContentLength: common.Int64(0), + ContentLength: int64Ptr(0), } _, err := s.Request(ctx, opts, nil, nil) @@ -476,25 +800,26 @@ func (s *Session) ValidateSession(ctx context.Context) error { Path: "/validate", ExtraHeaders: s.GetHeaders(map[string]string{}), RootURL: setupEndpoint, - ContentLength: common.Int64(0), + ContentLength: int64Ptr(0), } _, err := s.Request(ctx, opts, nil, &s.AccountInfo) if err != nil { return fmt.Errorf("validateSession failed: %w", err) } + s.needs2FA = false return nil } -// GetAuthHeaders returns the authentication headers for the session. -// Used for 2FA validation and trust requests to idmsa.apple.com. +// GetAuthHeaders returns the authentication headers for the session +// Used for 2FA validation and trust requests to idmsa.apple.com func (s *Session) GetAuthHeaders(overwrite map[string]string) map[string]string { headers := s.getSRPAuthHeaders() maps.Copy(headers, overwrite) return headers } -// GetHeaders Gets the authentication headers required for a request +// GetHeaders returns the authentication headers required for a request func (s *Session) GetHeaders(overwrite map[string]string) map[string]string { headers := GetCommonHeaders(map[string]string{}) headers["Cookie"] = s.GetCookieString() @@ -502,17 +827,26 @@ func (s *Session) GetHeaders(overwrite map[string]string) map[string]string { return headers } -// GetCookieString returns the cookie header string for the session. +// GetCookieString returns the cookie header string for the session func (s *Session) GetCookieString() string { - cookieHeader := "" - // we only care about name and value. + var b strings.Builder + first := true for _, cookie := range s.Cookies { - cookieHeader = cookieHeader + cookie.Name + "=" + cookie.Value + ";" + if cookie.Value == "" { + continue + } + if !first { + b.WriteString("; ") + } + first = false + b.WriteString(cookie.Name) + b.WriteByte('=') + b.WriteString(cookie.Value) } - return cookieHeader + return b.String() } -// GetCommonHeaders generates common HTTP headers with optional overwrite. +// GetCommonHeaders generates common HTTP headers with optional overwrite func GetCommonHeaders(overwrite map[string]string) map[string]string { headers := map[string]string{ "Content-Type": "application/json", @@ -524,32 +858,9 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string { return headers } -// MergeCookies merges two slices of http.Cookies, ensuring no duplicates are added. -func MergeCookies(left []*http.Cookie, right []*http.Cookie) ([]*http.Cookie, error) { - var hashes []string - for _, cookie := range right { - hashes = append(hashes, cookie.Raw) - } - for _, cookie := range left { - if !slices.Contains(hashes, cookie.Raw) { - right = append(right, cookie) - } - } - return right, nil -} +func int64Ptr(v int64) *int64 { return &v } -// GetCookiesForDomain filters the provided cookies based on the domain of the given URL. -func GetCookiesForDomain(url *url.URL, cookies []*http.Cookie) ([]*http.Cookie, error) { - var domainCookies []*http.Cookie - for _, cookie := range cookies { - if strings.HasSuffix(url.Host, cookie.Domain) { - domainCookies = append(domainCookies, cookie) - } - } - return domainCookies, nil -} - -// NewSession creates a new Session instance with default values. +// NewSession creates a new Session instance with default values func NewSession() *Session { session := &Session{ FrameID: strings.ToLower(uuid.New().String()), @@ -564,117 +875,20 @@ func NewSession() *Session { return session } -// AccountInfo represents an account info +// AccountInfo represents the subset of Apple's account response we actually use +// json.Unmarshal silently ignores extra fields in the response type AccountInfo struct { - DsInfo *ValidateDataDsInfo `json:"dsInfo"` - HasMinimumDeviceForPhotosWeb bool `json:"hasMinimumDeviceForPhotosWeb"` - ICDPEnabled bool `json:"iCDPEnabled"` - Webservices map[string]*webService `json:"webservices"` - PcsEnabled bool `json:"pcsEnabled"` - TermsUpdateNeeded bool `json:"termsUpdateNeeded"` - ConfigBag struct { - Urls struct { - AccountCreateUI string `json:"accountCreateUI"` - AccountLoginUI string `json:"accountLoginUI"` - AccountLogin string `json:"accountLogin"` - AccountRepairUI string `json:"accountRepairUI"` - DownloadICloudTerms string `json:"downloadICloudTerms"` - RepairDone string `json:"repairDone"` - AccountAuthorizeUI string `json:"accountAuthorizeUI"` - VettingURLForEmail string `json:"vettingUrlForEmail"` - AccountCreate string `json:"accountCreate"` - GetICloudTerms string `json:"getICloudTerms"` - VettingURLForPhone string `json:"vettingUrlForPhone"` - } `json:"urls"` - AccountCreateEnabled bool `json:"accountCreateEnabled"` - } `json:"configBag"` - HsaTrustedBrowser bool `json:"hsaTrustedBrowser"` - AppsOrder []string `json:"appsOrder"` - Version int `json:"version"` - IsExtendedLogin bool `json:"isExtendedLogin"` - PcsServiceIdentitiesIncluded bool `json:"pcsServiceIdentitiesIncluded"` - IsRepairNeeded bool `json:"isRepairNeeded"` - HsaChallengeRequired bool `json:"hsaChallengeRequired"` - RequestInfo struct { - Country string `json:"country"` - TimeZone string `json:"timeZone"` - Region string `json:"region"` - } `json:"requestInfo"` - PcsDeleted bool `json:"pcsDeleted"` - ICloudInfo struct { - SafariBookmarksHasMigratedToCloudKit bool `json:"SafariBookmarksHasMigratedToCloudKit"` - } `json:"iCloudInfo"` - Apps map[string]*ValidateDataApp `json:"apps"` + DsInfo *dsInfo `json:"dsInfo"` + Webservices map[string]*webService `json:"webservices"` + HsaChallengeRequired bool `json:"hsaChallengeRequired"` } -// ValidateDataDsInfo represents an validation info -type ValidateDataDsInfo struct { - HsaVersion int `json:"hsaVersion"` - LastName string `json:"lastName"` - ICDPEnabled bool `json:"iCDPEnabled"` - TantorMigrated bool `json:"tantorMigrated"` - Dsid string `json:"dsid"` - HsaEnabled bool `json:"hsaEnabled"` - IsHideMyEmailSubscriptionActive bool `json:"isHideMyEmailSubscriptionActive"` - IroncadeMigrated bool `json:"ironcadeMigrated"` - Locale string `json:"locale"` - BrZoneConsolidated bool `json:"brZoneConsolidated"` - ICDRSCapableDeviceList string `json:"ICDRSCapableDeviceList"` - IsManagedAppleID bool `json:"isManagedAppleID"` - IsCustomDomainsFeatureAvailable bool `json:"isCustomDomainsFeatureAvailable"` - IsHideMyEmailFeatureAvailable bool `json:"isHideMyEmailFeatureAvailable"` - ContinueOnDeviceEligibleDeviceInfo []string `json:"ContinueOnDeviceEligibleDeviceInfo"` - Gilligvited bool `json:"gilligvited"` - AppleIDAliases []any `json:"appleIdAliases"` - UbiquityEOLEnabled bool `json:"ubiquityEOLEnabled"` - IsPaidDeveloper bool `json:"isPaidDeveloper"` - CountryCode string `json:"countryCode"` - NotificationID string `json:"notificationId"` - PrimaryEmailVerified bool `json:"primaryEmailVerified"` - ADsID string `json:"aDsID"` - Locked bool `json:"locked"` - ICDRSCapableDeviceCount int `json:"ICDRSCapableDeviceCount"` - HasICloudQualifyingDevice bool `json:"hasICloudQualifyingDevice"` - PrimaryEmail string `json:"primaryEmail"` - AppleIDEntries []struct { - IsPrimary bool `json:"isPrimary"` - Type string `json:"type"` - Value string `json:"value"` - } `json:"appleIdEntries"` - GilliganEnabled bool `json:"gilligan-enabled"` - IsWebAccessAllowed bool `json:"isWebAccessAllowed"` - FullName string `json:"fullName"` - MailFlags struct { - IsThreadingAvailable bool `json:"isThreadingAvailable"` - IsSearchV2Provisioned bool `json:"isSearchV2Provisioned"` - SCKMail bool `json:"sCKMail"` - IsMppSupportedInCurrentCountry bool `json:"isMppSupportedInCurrentCountry"` - } `json:"mailFlags"` - LanguageCode string `json:"languageCode"` - AppleID string `json:"appleId"` - HasUnreleasedOS bool `json:"hasUnreleasedOS"` - AnalyticsOptInStatus bool `json:"analyticsOptInStatus"` - FirstName string `json:"firstName"` - ICloudAppleIDAlias string `json:"iCloudAppleIdAlias"` - NotesMigrated bool `json:"notesMigrated"` - BeneficiaryInfo struct { - IsBeneficiary bool `json:"isBeneficiary"` - } `json:"beneficiaryInfo"` - HasPaymentInfo bool `json:"hasPaymentInfo"` - PcsDelet bool `json:"pcsDelet"` - AppleIDAlias string `json:"appleIdAlias"` - BrMigrated bool `json:"brMigrated"` - StatusCode int `json:"statusCode"` - FamilyEligible bool `json:"familyEligible"` +// dsInfo holds the account metadata fields we read +type dsInfo struct { + HsaVersion int `json:"hsaVersion"` } -// ValidateDataApp represents an app -type ValidateDataApp struct { - CanLaunchWithOneFactor bool `json:"canLaunchWithOneFactor"` - IsQualifiedForBeta bool `json:"isQualifiedForBeta"` -} - -// WebService represents a web service +// webService represents a web service type webService struct { PcsRequired bool `json:"pcsRequired"` URL string `json:"url"` diff --git a/backend/iclouddrive/api/session_test.go b/backend/iclouddrive/api/session_test.go new file mode 100644 index 000000000..c3d06a37b --- /dev/null +++ b/backend/iclouddrive/api/session_test.go @@ -0,0 +1,40 @@ +package api + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractHeadersMergesCookies(t *testing.T) { + s := NewSession() + s.Cookies = []*http.Cookie{{Name: "existing", Value: "old"}} + + resp := &http.Response{Header: make(http.Header)} + resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "existing", Value: "new"}).String()) + resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "fresh", Value: "value"}).String()) + resp.Header.Set("X-Apple-Session-Token", "session-token") + + s.extractHeaders(resp) + + require.Len(t, s.Cookies, 2) + assert.Equal(t, "new", s.Cookies[0].Value) + assert.Equal(t, "fresh", s.Cookies[1].Name) + assert.Equal(t, "session-token", s.SessionToken) +} + +func TestExtractHeadersDeletesEmptyCookies(t *testing.T) { + s := NewSession() + s.Cookies = []*http.Cookie{{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: "stale"}, {Name: "keep", Value: "value"}} + + resp := &http.Response{Header: make(http.Header)} + resp.Header.Add("Set-Cookie", (&http.Cookie{Name: "X-APPLE-WEBAUTH-HSA-LOGIN", Value: ""}).String()) + + s.extractHeaders(resp) + + require.Len(t, s.Cookies, 1) + assert.Equal(t, "keep", s.Cookies[0].Name) + assert.Equal(t, "keep=value", s.GetCookieString()) +} diff --git a/backend/iclouddrive/api/srp.go b/backend/iclouddrive/api/srp.go index e2f7c7f66..9ca50eea4 100644 --- a/backend/iclouddrive/api/srp.go +++ b/backend/iclouddrive/api/srp.go @@ -46,10 +46,10 @@ type srpClient struct { } // newSRPClient generates a random 32-byte secret and computes the public value A. -func newSRPClient() *srpClient { +func newSRPClient() (*srpClient, error) { secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { - panic(fmt.Sprintf("srp: rand.Read failed: %v", err)) + return nil, fmt.Errorf("srp: rand.Read failed: %w", err) } a := new(big.Int).SetBytes(secret) @@ -60,7 +60,7 @@ func newSRPClient() *srpClient { a: a, A: A, k: k, - } + }, nil } // getABytes returns the padded public value A. @@ -71,16 +71,21 @@ func (c *srpClient) getABytes() []byte { // processChallenge computes the session key and proof values from the server challenge. // username is the Apple ID, derivedKey is the output of derivePassword, salt and B come // from the server's init response (raw bytes, already base64-decoded). -func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) { +func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) error { B := new(big.Int).SetBytes(serverB) - // Validate B + // Validate B (RFC 5054) if B.Cmp(big.NewInt(0)) <= 0 || B.Cmp(srpN) >= 0 { - panic("srp: invalid server-supplied B, must be 1..N-1") + return fmt.Errorf("srp: invalid server-supplied B, must be 1..N-1") } x := calculateX(salt, derivedKey) u := calculateU(c.A, B) + + // Validate u (RFC 5054) + if u.Sign() == 0 { + return fmt.Errorf("srp: calculated u is zero, aborting") + } S := calculateS(c.k, x, c.a, B, u) c.K = calculateK(S) @@ -88,6 +93,7 @@ func (c *srpClient) processChallenge(username, derivedKey, salt, serverB []byte) bBytes := padToN(B) c.M1 = calculateM1(username, salt, aBytes, bBytes, c.K) c.M2 = calculateM2(aBytes, c.M1, c.K) + return nil } // derivePassword performs Apple's password key derivation. @@ -98,10 +104,12 @@ func derivePassword(password string, salt []byte, iterations int, protocol strin var passInput string switch protocol { + case "s2k": + passInput = string(passHash[:]) case "s2k_fo": passInput = hex.EncodeToString(passHash[:]) - default: // "s2k" - passInput = string(passHash[:]) + default: + return nil, fmt.Errorf("unsupported SRP protocol: %q", protocol) } return pbkdf2.Key(sha256.New, passInput, salt, iterations, 32) } @@ -142,7 +150,7 @@ func getMultiplier() *big.Int { } // calculateX computes x = H(salt | H(":" | password)) -// Apple's variant: NoUserNameInX — username is omitted from the inner hash. +// Apple's variant: NoUserNameInX - username is omitted from the inner hash. // The "password" here is actually the derived key from derivePassword. func calculateX(salt, derivedKey []byte) *big.Int { h := srpHashFunc() diff --git a/backend/iclouddrive/api/srp_test.go b/backend/iclouddrive/api/srp_test.go index 3136eda55..5eec3c011 100644 --- a/backend/iclouddrive/api/srp_test.go +++ b/backend/iclouddrive/api/srp_test.go @@ -27,7 +27,7 @@ func TestPadToN(t *testing.T) { }) t.Run("preserves value that fills N", func(t *testing.T) { - // Use N itself — already 256 bytes + // Use N itself - already 256 bytes result := padToN(srpN) assert.Equal(t, srpNLenBytes, len(result)) assert.Equal(t, srpN.Bytes(), result) @@ -52,7 +52,7 @@ func TestGetMultiplier(t *testing.T) { k2 := getMultiplier() assert.Equal(t, 0, k.Cmp(k2), "multiplier must be deterministic") - // k = H(N | pad(g)) — verify by manual computation + // k = H(N | pad(g)) - verify by manual computation h := srpHashFunc() nBytes := srpN.Bytes() gBytes := srpG.Bytes() @@ -123,6 +123,12 @@ func TestDerivePassword(t *testing.T) { require.NoError(t, err) assert.Equal(t, 32, len(key)) }) + + t.Run("unknown protocol returns error", func(t *testing.T) { + _, err := derivePassword(password, salt, 1000, "unknown") + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported SRP protocol") + }) } func TestCalculateX(t *testing.T) { @@ -256,7 +262,8 @@ func TestCalculateM2(t *testing.T) { } func TestNewSRPClient(t *testing.T) { - client := newSRPClient() + client, err := newSRPClient() + require.NoError(t, err) // A must be non-zero and in range [1, N-1] assert.True(t, client.A.Sign() > 0, "A must be positive") @@ -273,7 +280,8 @@ func TestNewSRPClient(t *testing.T) { assert.Equal(t, 0, client.A.Cmp(recoveredA)) // Two clients should have different secrets (probabilistic, but 2^256 collision chance) - client2 := newSRPClient() + client2, err := newSRPClient() + require.NoError(t, err) assert.NotEqual(t, 0, client.a.Cmp(client2.a), "two clients must have different secrets") } @@ -315,7 +323,8 @@ func TestProcessChallenge(t *testing.T) { serverB := padToN(B) - client.processChallenge(username, derivedKey, salt, serverB) + err := client.processChallenge(username, derivedKey, salt, serverB) + require.NoError(t, err) // M1, M2, K must be populated assert.NotNil(t, client.M1) @@ -328,13 +337,14 @@ func TestProcessChallenge(t *testing.T) { // M1 and M2 must be different assert.NotEqual(t, client.M1, client.M2) - // Results must be deterministic — run again with same inputs + // Results must be deterministic - run again with same inputs client2 := &srpClient{ a: new(big.Int).Set(a), A: new(big.Int).Set(A), k: new(big.Int).Set(k), } - client2.processChallenge(username, derivedKey, salt, serverB) + err = client2.processChallenge(username, derivedKey, salt, serverB) + require.NoError(t, err) assert.Equal(t, client.M1, client2.M1, "M1 must be deterministic") assert.Equal(t, client.M2, client2.M2, "M2 must be deterministic") @@ -352,6 +362,28 @@ func TestProcessChallenge(t *testing.T) { assert.Equal(t, client.K, kServer, "client and server must derive the same session key") } +func TestProcessChallenge_InvalidB(t *testing.T) { + k := getMultiplier() + client := &srpClient{ + a: big.NewInt(42), + A: new(big.Int).Exp(srpG, big.NewInt(42), srpN), + k: k, + } + username := []byte("test@apple.com") + derivedKey := make([]byte, 32) + salt := []byte("salt") + + t.Run("B=0 rejected", func(t *testing.T) { + err := client.processChallenge(username, derivedKey, salt, padToN(big.NewInt(0))) + assert.Error(t, err) + }) + + t.Run("B=N rejected", func(t *testing.T) { + err := client.processChallenge(username, derivedKey, salt, padToN(srpN)) + assert.Error(t, err) + }) +} + func TestSRPGroupParameters(t *testing.T) { // Verify the 2048-bit prime starts with known hex prefix nHex := hex.EncodeToString(srpN.Bytes()) diff --git a/backend/iclouddrive/icloud.go b/backend/iclouddrive/icloud.go new file mode 100644 index 000000000..40ee6c949 --- /dev/null +++ b/backend/iclouddrive/icloud.go @@ -0,0 +1,495 @@ +//go:build !plan9 && !solaris + +// Package iclouddrive provides access to iCloud Drive and Photos +package iclouddrive + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "strconv" + "strings" + + "github.com/rclone/rclone/backend/iclouddrive/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/configstruct" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/lib/encoder" +) + +const configAuthSession = "_auth_session" + +// configAuthState holds session fields that must be preserved across +// Config() state machine steps (push 2FA and SMS both need it) +type configAuthState struct { + Scnt string `json:"s"` + SessionID string `json:"i"` + AuthAttributes string `json:"a"` + FrameID string `json:"f"` + SessionToken string `json:"t"` + ClientID string `json:"c"` + AccountCountry string `json:"ac"` + Phones []smsPhone `json:"p,omitempty"` +} + +type smsPhone struct { + ID int `json:"id"` + Num string `json:"n"` + Mode string `json:"m"` +} + +func saveAuthSession(m configmap.Mapper, s *api.Session, phones []smsPhone) { + st := configAuthState{ + Scnt: s.Scnt, + SessionID: s.SessionID, + AuthAttributes: s.AuthAttributes, + FrameID: s.FrameID, + SessionToken: s.SessionToken, + ClientID: s.ClientID, + AccountCountry: s.AccountCountry, + Phones: phones, + } + data, err := json.Marshal(st) + if err != nil { + fs.Debugf(nil, "iclouddrive: failed to marshal auth session: %v", err) + return + } + m.Set(configAuthSession, base64.StdEncoding.EncodeToString(data)) +} + +func loadAuthSession(m configmap.Mapper) (*configAuthState, error) { + raw, _ := m.Get(configAuthSession) + if raw == "" { + return nil, errors.New("auth session state lost, please reconfigure") + } + data, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("corrupt auth session state: %w", err) + } + var st configAuthState + if err := json.Unmarshal(data, &st); err != nil { + return nil, fmt.Errorf("invalid auth session state: %w", err) + } + return &st, nil +} + +func restoreAuthSession(icloud *api.Client, st *configAuthState) { + icloud.Session.Scnt = st.Scnt + icloud.Session.SessionID = st.SessionID + icloud.Session.AuthAttributes = st.AuthAttributes + icloud.Session.FrameID = st.FrameID + icloud.Session.SessionToken = st.SessionToken + icloud.Session.ClientID = st.ClientID + icloud.Session.AccountCountry = st.AccountCountry +} + +// resumeConfigClient recreates an API client and restores session state saved +// from a previous Config() step. Used across 2FA states to avoid re-authenticating +func resumeConfigClient(m configmap.Mapper, appleid, password, trustToken, clientID string, cookies []*http.Cookie) (*api.Client, *configAuthState, error) { + st, err := loadAuthSession(m) + if err != nil { + return nil, nil, err + } + icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil, "_config") + if err != nil { + return nil, nil, err + } + restoreAuthSession(icloud, st) + return icloud, st, nil +} + +// saveAuthCredentials persists trust token, cookies, and clears session state +// after successful 2FA validation +func saveAuthCredentials(m configmap.Mapper, icloud *api.Client, name string) { + m.Set(configTrustToken, icloud.Session.TrustToken) + m.Set(configCookies, icloud.Session.GetCookieString()) + m.Set(configAuthSession, "") + api.ClearCacheDir(name) +} + +// triggerSMSFlow handles phone selection and SMS triggering +// Single phone: triggers SMS immediately, returns code prompt +// Multiple phones: saves session + phone list, returns phone picker +func triggerSMSFlow(ctx context.Context, icloud *api.Client, phones []api.TrustedPhoneNumber, m configmap.Mapper) (*fs.ConfigOut, error) { + smsPhones := make([]smsPhone, len(phones)) + for i, p := range phones { + mode := p.PushMode + if mode == "" { + mode = "sms" + } + smsPhones[i] = smsPhone{ID: p.ID, Num: p.NumberWithDialCode, Mode: mode} + } + + if len(phones) == 1 { + p := smsPhones[0] + if err := icloud.Session.RequestSMSCode(ctx, p.ID, p.Mode); err != nil { + return nil, fmt.Errorf("failed to send SMS code: %w", err) + } + saveAuthSession(m, icloud.Session, nil) + nextState := fmt.Sprintf("2fa_sms_%d_%s", p.ID, p.Mode) + return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", p.Num)) + } + + // Multiple phones - save session and present picker + saveAuthSession(m, icloud.Session, smsPhones) + items := make([]fs.OptionExample, len(smsPhones)) + for i, p := range smsPhones { + items[i] = fs.OptionExample{ + Value: fmt.Sprintf("%d_%s", p.ID, p.Mode), + Help: p.Num, + } + } + return fs.ConfigChooseExclusiveFixed("2fa_sms_select", "config_2fa_phone", "Select phone number for SMS verification", items) +} + +const ( + configService = "service" + + // Service types + serviceDrive = "drive" + servicePhotos = "photos" +) + +// ServiceOptions defines the configuration for service selection +type ServiceOptions struct { + Service string `config:"service"` +} + +// Register with rclone +func init() { + fs.Register(&fs.RegInfo{ + Name: "iclouddrive", + Description: "iCloud Drive and Photos", + Config: Config, + NewFs: NewServiceFs, + MetadataInfo: &fs.MetadataInfo{ + System: map[string]fs.MetadataHelp{ + "width": { + Help: "Image width in pixels", + Type: "int", + ReadOnly: true, + }, + "height": { + Help: "Image height in pixels", + Type: "int", + ReadOnly: true, + }, + "added-time": { + Help: "Time the item was added to the iCloud library", + Type: "RFC 3339", + Example: "2006-01-02T15:04:05Z", + ReadOnly: true, + }, + "favorite": { + Help: "Whether the item is marked as favorite", + Type: "bool", + ReadOnly: true, + }, + "hidden": { + Help: "Whether the item is hidden", + Type: "bool", + ReadOnly: true, + }, + }, + Help: "Metadata is read-only and available for the Photos service only.", + }, + Options: []fs.Option{{ + Name: configService, + Help: "iCloud service to use.", + Required: true, + Default: serviceDrive, + Examples: []fs.OptionExample{{ + Value: serviceDrive, + Help: "iCloud Drive", + }, { + Value: servicePhotos, + Help: "iCloud Photos", + }}, + }, { + Name: configAppleID, + Help: "Apple ID.", + Required: true, + Sensitive: true, + }, { + Name: configPassword, + Help: "Password.", + Required: true, + IsPassword: true, + Sensitive: true, + }, { + Name: configTrustToken, + Help: "Trust token for session authentication.", + IsPassword: false, + Required: false, + Sensitive: true, + Hide: fs.OptionHideBoth, + }, { + Name: configCookies, + Help: "Session cookies.", + Required: false, + Advanced: false, + Sensitive: true, + Hide: fs.OptionHideBoth, + }, { + Name: configClientID, + Help: "Client ID for iCloud API access.", + Required: false, + Advanced: true, + Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d", + }, { + Name: config.ConfigEncoding, + Help: config.ConfigEncodingHelp, + Advanced: true, + Default: (encoder.Display | + encoder.EncodeBackSlash | + encoder.EncodeSlash | + encoder.EncodeInvalidUtf8), + }}, + }) +} + +// Config handles the authentication and configuration flow +func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { + var err error + appleid, _ := m.Get(configAppleID) + if appleid == "" { + return nil, errors.New("an Apple ID is required") + } + + password, _ := m.Get(configPassword) + if password != "" { + password, err = obscure.Reveal(password) + if err != nil { + return nil, err + } + } + + trustToken, _ := m.Get(configTrustToken) + cookieRaw, _ := m.Get(configCookies) + clientID, _ := m.Get(configClientID) + cookies := ReadCookies(cookieRaw) + + switch { + case config.State == "": + // Force fresh SRP authentication - ignore stale trust token and cookies + // so that reconnect always prompts for 2FA + m.Set(configAuthSession, "") + icloud, err := api.New(appleid, password, "", clientID, nil, nil, "_config") + if err != nil { + return nil, err + } + if err := icloud.Authenticate(ctx); err != nil { + return nil, err + } + m.Set(configCookies, icloud.Session.GetCookieString()) + if icloud.Session.Requires2FA() { + // Check if user has no trusted devices - auto-trigger SMS if so + authState, err := icloud.Session.GetAuthState(ctx) + if err == nil && authState.NoTrustedDevices && len(authState.TrustedPhoneNumbers) > 0 { + return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m) + } + // Explicitly request push to trusted devices - required for iOS 26.4+ + // where the SRP 409 no longer auto-pushes. GET /appleauth/auth above + // may also trigger a push (cosmetic double on pre-26.4, harmless) + if err := icloud.Session.RequestPushNotification(ctx); err != nil { + fs.Debugf(nil, "iclouddrive: push notification request failed (SMS fallback available): %v", err) + } else { + fs.Debugf(nil, "iclouddrive: push notification requested to trusted devices") + } + // Save session state so 2fa_do can validate without re-authenticating + // Push codes are account-scoped so session reuse is not strictly required, + // but it avoids a redundant SRP roundtrip and a second push on pre-26.4 + saveAuthSession(m, icloud.Session, nil) + return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: enter your 2FA code or type 'sms' for a text message") + } + // Auth succeeded without 2FA - save updated credentials and clear stale cache + m.Set(configTrustToken, icloud.Session.TrustToken) + api.ClearCacheDir(name) + return nil, nil + + case config.State == "2fa_do": + code := config.Result + if code == "" { + return fs.ConfigError("authenticate", "2FA codes can't be blank") + } + + // Restore session from initial sign-in instead of re-authenticating + // This avoids a redundant SRP roundtrip and extra push on pre-26.4 + icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies) + if err != nil { + return nil, err + } + + if strings.EqualFold(code, "sms") { + authState, err := icloud.Session.GetAuthState(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get trusted phone numbers: %w", err) + } + if len(authState.TrustedPhoneNumbers) == 0 { + return nil, errors.New("no trusted phone numbers on this account") + } + return triggerSMSFlow(ctx, icloud, authState.TrustedPhoneNumbers, m) + } + + if err := icloud.Session.Validate2FACode(ctx, code); err != nil { + return nil, err + } + saveAuthCredentials(m, icloud, name) + return nil, nil + + case config.State == "2fa_sms_select": + // User selected a phone from ConfigChooseExclusiveFixed (value: "ID_mode") + idStr, mode, _ := strings.Cut(config.Result, "_") + phoneID, err := strconv.Atoi(idStr) + if err != nil { + m.Set(configAuthSession, "") + return nil, fmt.Errorf("invalid phone selection %q", config.Result) + } + if mode == "" { + mode = "sms" + } + icloud, smsState, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies) + if err != nil { + return nil, err + } + // Find the selected phone's display number + var phoneNum string + for _, p := range smsState.Phones { + if p.ID == phoneID { + phoneNum = p.Num + break + } + } + + if err := icloud.Session.RequestSMSCode(ctx, phoneID, mode); err != nil { + m.Set(configAuthSession, "") + return nil, fmt.Errorf("failed to send SMS code: %w", err) + } + saveAuthSession(m, icloud.Session, nil) + nextState := fmt.Sprintf("2fa_sms_%d_%s", phoneID, mode) + return fs.ConfigInput(nextState, "config_2fa_sms", fmt.Sprintf("Enter the verification code sent to %s", phoneNum)) + + case strings.HasPrefix(config.State, "2fa_sms_"): + code := config.Result + if code == "" { + return fs.ConfigError("authenticate", "SMS code can't be blank") + } + // State encodes phone ID and mode: "2fa_sms__" + suffix := strings.TrimPrefix(config.State, "2fa_sms_") + idStr, mode, _ := strings.Cut(suffix, "_") + phoneID, err := strconv.Atoi(idStr) + if err != nil { + return nil, fmt.Errorf("invalid phone ID in state %q: %w", config.State, err) + } + if mode == "" { + mode = "sms" + } + + icloud, _, err := resumeConfigClient(m, appleid, password, trustToken, clientID, cookies) + if err != nil { + return nil, err + } + + if err := icloud.Session.ValidateSMSCode(ctx, code, phoneID, mode); err != nil { + m.Set(configAuthSession, "") + return nil, err + } + saveAuthCredentials(m, icloud, name) + return nil, nil + + default: + return nil, fmt.Errorf("unknown state %q", config.State) + } +} + +// newICloudClient parses options, authenticates, and returns a ready client +// Shared between NewFs (Drive) and NewFsPhotos (Photos) to avoid duplication +func newICloudClient(ctx context.Context, name string, m configmap.Mapper) (*api.Client, *Options, error) { + opt := new(Options) + err := configstruct.Set(m, opt) + if err != nil { + return nil, nil, err + } + + if opt.Password != "" { + opt.Password, err = obscure.Reveal(opt.Password) + if err != nil { + return nil, nil, fmt.Errorf("couldn't decrypt user password: %w", err) + } + } + + if opt.TrustToken == "" { + return nil, nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name) + } + + cookies := ReadCookies(opt.Cookies) + + callback := func(session *api.Session) { + m.Set(configCookies, session.GetCookieString()) + } + + icloud, err := api.New( + opt.AppleID, + opt.Password, + opt.TrustToken, + opt.ClientID, + cookies, + callback, + name, + ) + if err != nil { + return nil, nil, err + } + + if err := icloud.Authenticate(ctx); err != nil { + return nil, nil, err + } + + if icloud.Session.Requires2FA() { + return nil, nil, errors.New("trust token expired, please reauth") + } + + return icloud, opt, nil +} + +// disconnectClient clears authentication state and removes disk caches +// Shared between Drive Fs and Photos Fs Disconnect() implementations +func disconnectClient(m configmap.Mapper, icloud *api.Client) error { + m.Set(configTrustToken, "") + m.Set(configCookies, "") + m.Set(configAuthSession, "") + return os.RemoveAll(icloud.CacheDir()) +} + +// NewServiceFs creates a filesystem instance for the selected service +func NewServiceFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + // Parse the service selection + opt := new(ServiceOptions) + err := configstruct.Set(m, opt) + if err != nil { + return nil, err + } + + // Set default service if not specified + if opt.Service == "" { + opt.Service = serviceDrive + } + + // Route to the appropriate backend + switch opt.Service { + case serviceDrive: + // Create Drive filesystem + return NewFs(ctx, name, root, m) + case servicePhotos: + // Create Photos filesystem + return NewFsPhotos(ctx, name, root, m) + default: + return nil, fmt.Errorf("invalid service selection: %s (must be 'drive' or 'photos')", opt.Service) + } +} diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.go index 1a51b9ab8..135ec75e9 100644 --- a/backend/iclouddrive/iclouddrive.go +++ b/backend/iclouddrive/iclouddrive.go @@ -9,17 +9,13 @@ import ( "path" "errors" - "fmt" "io" "net/http" "strings" "time" "github.com/rclone/rclone/fs" - "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/configmap" - "github.com/rclone/rclone/fs/config/configstruct" - "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/backend/iclouddrive/api" @@ -51,61 +47,10 @@ const ( decayConstant = 2 ) -// Register with Fs -func init() { - fs.Register(&fs.RegInfo{ - Name: "iclouddrive", - Description: "iCloud Drive", - Config: Config, - NewFs: NewFs, - Options: []fs.Option{{ - Name: configAppleID, - Help: "Apple ID.", - Required: true, - Sensitive: true, - }, { - Name: configPassword, - Help: "Password.", - Required: true, - IsPassword: true, - Sensitive: true, - }, { - Name: configTrustToken, - Help: "Trust token (internal use)", - IsPassword: false, - Required: false, - Sensitive: true, - Hide: fs.OptionHideBoth, - }, { - Name: configCookies, - Help: "cookies (internal use only)", - Required: false, - Advanced: false, - Sensitive: true, - Hide: fs.OptionHideBoth, - }, { - Name: configClientID, - Help: "Client id", - Required: false, - Advanced: true, - Default: "d39ba9916b7251055b22c7f910e2ea796ee65e98b2ddecea8f5dde8d9d1a815d", - }, { - Name: config.ConfigEncoding, - Help: config.ConfigEncodingHelp, - Advanced: true, - Default: (encoder.Display | - //encoder.EncodeDot | - encoder.EncodeBackSlash | - encoder.EncodeInvalidUtf8), - }}, - }) -} - // Options defines the configuration for this backend type Options struct { AppleID string `config:"apple_id"` Password string `config:"password"` - Photos bool `config:"photos"` TrustToken string `config:"trust_token"` Cookies string `config:"cookies"` ClientID string `config:"client_id"` @@ -118,6 +63,7 @@ type Fs struct { root string // the path we are working on. rootID string opt Options // parsed config options + m configmap.Mapper // config map for persisting auth state features *fs.Features // optional features dirCache *dircache.DirCache // Map of directory path to directory id icloud *api.Client @@ -139,72 +85,6 @@ type Object struct { downloadURL string } -// Config configures the iCloud remote. -func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { - var err error - appleid, _ := m.Get(configAppleID) - if appleid == "" { - return nil, errors.New("a apple ID is required") - } - - password, _ := m.Get(configPassword) - if password != "" { - password, err = obscure.Reveal(password) - if err != nil { - return nil, err - } - } - - trustToken, _ := m.Get(configTrustToken) - cookieRaw, _ := m.Get(configCookies) - clientID, _ := m.Get(configClientID) - cookies := ReadCookies(cookieRaw) - - switch config.State { - case "": - icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil) - if err != nil { - return nil, err - } - if err := icloud.Authenticate(ctx); err != nil { - return nil, err - } - m.Set(configCookies, icloud.Session.GetCookieString()) - if icloud.Session.Requires2FA() { - return fs.ConfigInput("2fa_do", "config_2fa", "Two-factor authentication: please enter your 2FA code") - } - return nil, nil - case "2fa_do": - code := config.Result - if code == "" { - return fs.ConfigError("authenticate", "2FA codes can't be blank") - } - - icloud, err := api.New(appleid, password, trustToken, clientID, cookies, nil) - if err != nil { - return nil, err - } - if err := icloud.SignIn(ctx); err != nil { - return nil, err - } - - if err := icloud.Session.Validate2FACode(ctx, code); err != nil { - return nil, err - } - - m.Set(configTrustToken, icloud.Session.TrustToken) - m.Set(configCookies, icloud.Session.GetCookieString()) - return nil, nil - - case "2fa_error": - if config.Result == "true" { - return fs.ConfigGoto("2fa") - } - return nil, errors.New("2fa authentication failed") - } - return nil, fmt.Errorf("unknown state %q", config.State) -} - // find item by path. Will not return any children for the item func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, found bool, err error) { var resp *http.Response @@ -797,51 +677,11 @@ func retryResultUnknown(ctx context.Context, resp *http.Response, err error) (bo // NewFs constructs an Fs from the path, container:path func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { - // Parse config into Options struct - opt := new(Options) - err := configstruct.Set(m, opt) + icloud, opt, err := newICloudClient(ctx, name, m) if err != nil { return nil, err } - if opt.Password != "" { - var err error - opt.Password, err = obscure.Reveal(opt.Password) - if err != nil { - return nil, fmt.Errorf("couldn't decrypt user password: %w", err) - } - } - - if opt.TrustToken == "" { - return nil, fmt.Errorf("missing icloud trust token: try refreshing it with \"rclone config reconnect %s:\"", name) - } - - cookies := ReadCookies(opt.Cookies) - - callback := func(session *api.Session) { - m.Set(configCookies, session.GetCookieString()) - } - - icloud, err := api.New( - opt.AppleID, - opt.Password, - opt.TrustToken, - opt.ClientID, - cookies, - callback, - ) - if err != nil { - return nil, err - } - - if err := icloud.Authenticate(ctx); err != nil { - return nil, err - } - - if icloud.Session.Requires2FA() { - return nil, errors.New("trust token expired, please reauth") - } - root = strings.Trim(root, "/") f := &Fs{ @@ -850,6 +690,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e icloud: icloud, rootID: "FOLDER::com.apple.CloudDocs::root", opt: *opt, + m: m, pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), } f.features = (&fs.Features{ @@ -1164,9 +1005,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return nil } +// Disconnect clears authentication state and removes disk caches +func (f *Fs) Disconnect(ctx context.Context) error { + return disconnectClient(f.m, f.icloud) +} + // Check interfaces are satisfied var ( _ fs.Fs = &Fs{} + _ fs.Disconnecter = (*Fs)(nil) _ fs.Mover = (*Fs)(nil) _ fs.Purger = (*Fs)(nil) _ fs.DirMover = (*Fs)(nil) diff --git a/backend/iclouddrive/icloudphotos.go b/backend/iclouddrive/icloudphotos.go new file mode 100644 index 000000000..922333f98 --- /dev/null +++ b/backend/iclouddrive/icloudphotos.go @@ -0,0 +1,892 @@ +//go:build !plan9 && !solaris + +package iclouddrive + +import ( + "context" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + "sync" + "time" + + "github.com/rclone/rclone/backend/iclouddrive/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/fshttp" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/list" + "github.com/rclone/rclone/lib/dircache" + "github.com/rclone/rclone/lib/pacer" +) + +const rootID = "photos-root" + +// PhotosFs represents a remote iCloud Photos server +type PhotosFs struct { + name string + root string + opt Options + features *fs.Features + icloud *api.Client + m configmap.Mapper + pacer *fs.Pacer + dirCache *dircache.DirCache + httpClient *http.Client + startTime time.Time + mu sync.Mutex + photos *api.PhotosService +} + +// PhotosObject describes an iCloud Photos object +type PhotosObject struct { + fs *PhotosFs + remote string + size int64 + modTime time.Time + masterID string // CloudKit recordName for fresh URL lookup (master or asset depending on resource) + zone string // CloudKit zone (e.g. "PrimarySync") + resourceKey string // CloudKit resource field (resOriginalRes or resOriginalVidComplRes) + width int + height int + addedDate int64 + isFavorite bool + isHidden bool +} + +// NewFsPhotos constructs an Fs for Photos from the path, container:path +func NewFsPhotos(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + icloud, opt, err := newICloudClient(ctx, name, m) + if err != nil { + return nil, err + } + + root = strings.Trim(root, "/") + + f := &PhotosFs{ + name: name, + root: root, + opt: *opt, + icloud: icloud, + m: m, + pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), + httpClient: fshttp.NewClient(ctx), + startTime: time.Now(), + } + + f.features = (&fs.Features{ + CanHaveEmptyDirectories: true, + PartialUploads: false, + ReadMimeType: false, + ReadMetadata: true, + }).Fill(ctx, f) + + f.dirCache = dircache.New(root, rootID, f) + + err = f.dirCache.FindRoot(ctx, false) + if err != nil { + // Check if root points to a file (e.g. PrimarySync/Videos/file.mp4) + newRoot, remote := dircache.SplitPath(root) + tempF := &PhotosFs{ + name: f.name, + root: newRoot, + opt: f.opt, + features: f.features, + icloud: f.icloud, + pacer: f.pacer, + httpClient: f.httpClient, + startTime: f.startTime, + photos: f.photos, // nil here; photosService() creates lazily on first use + } + tempF.dirCache = dircache.New(newRoot, rootID, tempF) + if err2 := tempF.dirCache.FindRoot(ctx, false); err2 != nil { + return f, nil + } + _, err2 := tempF.NewObject(ctx, remote) + if err2 != nil { + return f, nil + } + // Root is a file - adjust f to parent dir, signal ErrorIsFile + f.root = newRoot + f.dirCache = tempF.dirCache + return f, fs.ErrorIsFile + } + + return f, nil +} + +// photosService returns the PhotosService, creating it on first call +func (f *PhotosFs) photosService(ctx context.Context) (*api.PhotosService, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.photos == nil { + var err error + f.photos, err = api.NewPhotosService(ctx, f.icloud, f.pacer, shouldRetry) + if err != nil { + return nil, err + } + } + return f.photos, nil +} + +// Name of the remote (as passed into NewFs) +func (f *PhotosFs) Name() string { + return f.name +} + +// Root of the remote (as passed into NewFs) +func (f *PhotosFs) Root() string { + return f.opt.Enc.ToStandardPath(f.root) +} + +// String converts this Fs to a string +func (f *PhotosFs) String() string { + return fmt.Sprintf("iCloud Photos root '%s'", f.root) +} + +// Precision of the object storage system +func (f *PhotosFs) Precision() time.Duration { + return time.Second +} + +// Hashes returns the supported hash sets +func (f *PhotosFs) Hashes() hash.Set { + return hash.Set(hash.None) +} + +// Features returns the optional features of this Fs +func (f *PhotosFs) Features() *fs.Features { + return f.features +} + +// List the objects and directories in dir into entries +func (f *PhotosFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + photosService, err := f.photosService(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get photos service: %w", err) + } + + dirID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return nil, err + } + + switch { + case dirID == rootID: + // List libraries + libraries, err := photosService.GetLibraries(ctx) + if err != nil { + return nil, err + } + + albumCounts, err := photosService.GetLibraryAlbumCounts(ctx) + if err != nil { + fs.Debugf(f, "Failed to get library album counts: %v", err) + } + + for libraryName := range libraries { + count := int64(-1) + if albumCounts != nil { + if c, ok := albumCounts[libraryName]; ok { + count = c + } + } + d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(libraryName)), f.startTime).SetItems(count) + entries = append(entries, d) + } + + case strings.HasPrefix(dirID, "lib:"): + // List albums in a library + libraryName := strings.TrimPrefix(dirID, "lib:") + libraries, err := photosService.GetLibraries(ctx) + if err != nil { + return nil, err + } + + library, exists := libraries[libraryName] + if !exists { + return nil, fs.ErrorDirNotFound + } + + albums, err := library.GetAlbums(ctx) + if err != nil { + return nil, err + } + + counts, err := library.GetAlbumCounts(ctx) + if err != nil { + fs.Debugf(f, "Failed to get album counts for library %q: %v", libraryName, err) + } + + for albumName := range albums { + count := int64(-1) + if counts != nil { + if c, ok := counts[albumName]; ok { + count = c + } + } + d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(albumName)), f.startTime).SetItems(count) + entries = append(entries, d) + } + + case strings.HasPrefix(dirID, "album:"): + // List contents of an album or folder + libraryName, albumPath, ok := parseAlbumDirID(dirID) + if !ok { + return nil, fs.ErrorDirNotFound + } + + album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath) + if err != nil { + return nil, err + } + + // Folders list their child albums as subdirectories + if album.IsFolder { + for childName, child := range album.Children { + d := fs.NewDir(path.Join(dir, f.opt.Enc.ToStandardName(childName)), f.startTime) + if child.IsFolder { + d.SetItems(int64(len(child.Children))) + } + entries = append(entries, d) + } + return entries, nil + } + + photos, err := album.GetPhotos(ctx) + if err != nil { + return nil, err + } + + for _, photo := range photos { + if photo.Filename == "" { + continue + } + encodedName := f.opt.Enc.ToStandardName(photo.Filename) + remotePath := encodedName + if dir != "" { + remotePath = path.Join(dir, encodedName) + } + o := f.newPhotosObject(remotePath, photo, libraryName) + entries = append(entries, o) + } + + default: + return nil, fs.ErrorDirNotFound + } + + return entries, nil +} + +// NewObject finds the Object at remote. If it can't be found +// it returns the error fs.ErrorObjectNotFound +func (f *PhotosFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + dir, leaf := dircache.SplitPath(remote) + filename := f.opt.Enc.FromStandardName(leaf) + + dirID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + fs.Debugf(f, "NewObject(%s): FindDir: %v", remote, err) + return nil, fs.ErrorObjectNotFound + } + + if !strings.HasPrefix(dirID, "album:") { + return nil, fs.ErrorObjectNotFound + } + + libraryName, albumPath, ok := parseAlbumDirID(dirID) + if !ok { + return nil, fs.ErrorObjectNotFound + } + + photosService, err := f.photosService(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get photos service: %w", err) + } + + album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath) + if err != nil || album.IsFolder { + fs.Debugf(f, "NewObject(%s): album resolve failed or is folder", remote) + return nil, fs.ErrorObjectNotFound + } + + photo, err := album.GetPhotoByName(ctx, filename) + if err != nil { + fs.Debugf(f, "NewObject(%s): %v", remote, err) + return nil, fs.ErrorObjectNotFound + } + + return f.newPhotosObject(remote, photo, libraryName), nil +} + +// newPhotosObject creates a PhotosObject from a Photo and zone +func (f *PhotosFs) newPhotosObject(remote string, photo *api.Photo, zone string) *PhotosObject { + return &PhotosObject{ + fs: f, + remote: remote, + size: photo.Size, + modTime: time.UnixMilli(photo.AssetDate), + masterID: photo.ID, + zone: zone, + resourceKey: photo.ResourceKey, + width: photo.Width, + height: photo.Height, + addedDate: photo.AddedDate, + isFavorite: photo.IsFavorite, + isHidden: photo.IsHidden, + } +} + +// Put is not supported for this read-only backend +func (f *PhotosFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + return nil, fs.ErrorNotImplemented +} + +// Mkdir is not supported for this read-only backend +func (f *PhotosFs) Mkdir(ctx context.Context, dir string) error { + return fs.ErrorNotImplemented +} + +// Rmdir is not supported for this read-only backend +func (f *PhotosFs) Rmdir(ctx context.Context, dir string) error { + return fs.ErrorNotImplemented +} + +// FindLeaf finds a directory of name leaf in the folder with ID pathID +func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { + decodedLeaf := f.opt.Enc.FromStandardName(leaf) + + photosService, err := f.photosService(ctx) + if err != nil { + return "", false, fmt.Errorf("failed to get photos service: %w", err) + } + + if pathID == rootID { + libraries, err := photosService.GetLibraries(ctx) + if err != nil { + return "", false, err + } + + if _, exists := libraries[decodedLeaf]; exists { + return "lib:" + decodedLeaf, true, nil + } + return "", false, nil + } + + if strings.HasPrefix(pathID, "lib:") { + libraryName := strings.TrimPrefix(pathID, "lib:") + libraries, err := photosService.GetLibraries(ctx) + if err != nil { + return "", false, err + } + + library, exists := libraries[libraryName] + if !exists { + return "", false, fs.ErrorDirNotFound + } + + albums, err := library.GetAlbums(ctx) + if err != nil { + return "", false, err + } + + if _, exists := albums[decodedLeaf]; exists { + albumID := "album:" + libraryName + ":" + decodedLeaf + return albumID, true, nil + } + return "", false, nil + } + + if strings.HasPrefix(pathID, "album:") { + // Check if this is a folder containing child albums + libraryName, albumPath, ok := parseAlbumDirID(pathID) + if !ok { + return "", false, nil + } + + album, err := f.resolveAlbum(ctx, photosService, libraryName, albumPath) + if err != nil { + return "", false, nil + } + + if album.IsFolder { + if _, exists := album.Children[decodedLeaf]; exists { + childID := "album:" + libraryName + ":" + albumPath + "/" + decodedLeaf + return childID, true, nil + } + } + return "", false, nil + } + + return "", false, nil +} + +// CreateDir makes a directory with pathID as parent and name leaf +func (f *PhotosFs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { + return "", fs.ErrorNotImplemented +} + +// Fs returns the parent Fs +func (o *PhotosObject) Fs() fs.Info { + return o.fs +} + +// Return a string version +func (o *PhotosObject) String() string { + if o == nil { + return "" + } + return o.remote +} + +// Remote returns the remote path +func (o *PhotosObject) Remote() string { + return o.remote +} + +// ModTime returns the modification time of the object +func (o *PhotosObject) ModTime(ctx context.Context) time.Time { + return o.modTime +} + +// Size returns the size of an object in bytes +func (o *PhotosObject) Size() int64 { + return o.size +} + +// Storable returns a boolean as to whether this object is storable +func (o *PhotosObject) Storable() bool { + return true +} + +// Hash returns the hash of an object returning a lowercase hex string +func (o *PhotosObject) Hash(ctx context.Context, ty hash.Type) (string, error) { + return "", hash.ErrUnsupported +} + +// SetModTime sets the modification time of the object +func (o *PhotosObject) SetModTime(ctx context.Context, modTime time.Time) error { + return fs.ErrorCantSetModTime +} + +// Metadata returns metadata for the photo object +func (o *PhotosObject) Metadata(ctx context.Context) (fs.Metadata, error) { + metadata := make(fs.Metadata, 5) + if o.width > 0 { + metadata["width"] = strconv.Itoa(o.width) + } + if o.height > 0 { + metadata["height"] = strconv.Itoa(o.height) + } + if o.addedDate > 0 { + metadata["added-time"] = time.UnixMilli(o.addedDate).UTC().Format(time.RFC3339) + } + metadata["favorite"] = strconv.FormatBool(o.isFavorite) + metadata["hidden"] = strconv.FormatBool(o.isHidden) + return metadata, nil +} + +// Open an object for read, fetching a fresh download URL via records/lookup +func (o *PhotosObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { + photosService, err := o.fs.photosService(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get photos service: %w", err) + } + + downloadURL, err := photosService.LookupDownloadURL(ctx, o.masterID, o.zone, o.resourceKey) + if err != nil { + return nil, fmt.Errorf("failed to get download URL for %q: %w", o.remote, err) + } + + fs.FixRangeOption(options, o.size) + + var resp *http.Response + err = o.fs.pacer.Call(func() (bool, error) { + if resp != nil { + _ = resp.Body.Close() + resp = nil + } + req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil) + if err != nil { + return false, fmt.Errorf("failed to create request: %w", err) + } + fs.OpenOptionAddHTTPHeaders(req.Header, options) + + resp, err = o.fs.httpClient.Do(req) + return shouldRetry(ctx, resp, err) + }) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + _ = resp.Body.Close() + return nil, fmt.Errorf("download %q failed: %s", o.remote, resp.Status) + } + + return resp.Body, nil +} + +// Update is not supported for this read-only backend +func (o *PhotosObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { + return fs.ErrorNotImplemented +} + +// Remove is not supported for this read-only backend +func (o *PhotosObject) Remove(ctx context.Context) error { + return fs.ErrorNotImplemented +} + +// resolveAlbum finds an album by path, traversing folder hierarchy +// albumPath can be "AlbumName" or "FolderName/AlbumName" for nested albums +func (f *PhotosFs) resolveAlbum(ctx context.Context, ps *api.PhotosService, libraryName, albumPath string) (*api.Album, error) { + libraries, err := ps.GetLibraries(ctx) + if err != nil { + return nil, err + } + + library, exists := libraries[libraryName] + if !exists { + return nil, fs.ErrorDirNotFound + } + + albums, err := library.GetAlbums(ctx) + if err != nil { + return nil, err + } + + return resolveAlbumPath(albums, albumPath) +} + +// resolveAlbumPath finds an album inside an album tree by slash-separated path +func resolveAlbumPath(albums map[string]*api.Album, albumPath string) (*api.Album, error) { + parts := strings.Split(albumPath, "/") + current := albums + for i, part := range parts { + album, exists := current[part] + if !exists { + return nil, fs.ErrorDirNotFound + } + if i == len(parts)-1 { + return album, nil + } + if !album.IsFolder || album.Children == nil { + return nil, fs.ErrorDirNotFound + } + current = album.Children + } + return nil, fs.ErrorDirNotFound +} + +// parseAlbumDirID extracts library and album names from "album:lib:album" dirID +func parseAlbumDirID(dirID string) (libraryName, albumName string, ok bool) { + parts := strings.SplitN(strings.TrimPrefix(dirID, "album:"), ":", 2) + if len(parts) != 2 { + return "", "", false + } + return parts[0], parts[1], true +} + +// DirCacheFlush resets the directory cache - used in testing as an +// optional interface +func (f *PhotosFs) DirCacheFlush() { + f.dirCache.ResetRoot() + // Also flush API-layer caches so albums/photos are re-fetched + // DirCacheFlusher interface has no ctx param; use a bounded timeout + // to avoid hanging indefinitely if photosService triggers first init + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if ps, err := f.photosService(ctx); err == nil { + ps.FlushCaches() + } +} + +// listRAlbumJob represents an album to be listed by a ListR worker +type listRAlbumJob struct { + album *api.Album + zone string + dirPath string // path relative to Fs root (e.g. "Videos" when f.root="PrimarySync") +} + +// ListR lists the objects and directories of the Fs starting from dir +// recursively into out, calling callback for each batch of entries +// Albums are listed in parallel using a goroutine pool +func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) error { + ps, err := f.photosService(ctx) + if err != nil { + return err + } + + helper := list.NewHelper(callback) + var mu sync.Mutex // protects helper (not thread-safe) + + addEntry := func(entry fs.DirEntry) error { + mu.Lock() + defer mu.Unlock() + return helper.Add(entry) + } + + // Resolve dir through dirCache - this respects f.root + dirID, err := f.dirCache.FindDir(ctx, dir, false) + if err != nil { + return err + } + + // Collect album jobs, scoped to the resolved directory + var jobs []listRAlbumJob + + // collectAlbumJobs emits directory entries for albums/folders under basePath + // and collects leaf albums as jobs for parallel photo listing + // Recurses into folders to handle arbitrary nesting depth + var collectAlbumJobs func(albums map[string]*api.Album, zone, basePath string) error + collectAlbumJobs = func(albums map[string]*api.Album, zone, basePath string) error { + for albumName, album := range albums { + albumPath := path.Join(basePath, f.opt.Enc.ToStandardName(albumName)) + if err := addEntry(fs.NewDir(albumPath, f.startTime)); err != nil { + return err + } + if album.IsFolder { + if err := collectAlbumJobs(album.Children, zone, albumPath); err != nil { + return err + } + } else { + jobs = append(jobs, listRAlbumJob{album: album, zone: zone, dirPath: albumPath}) + } + } + return nil + } + + switch { + case dirID == rootID: + // Root level - emit libraries, then all albums + libraries, err := ps.GetLibraries(ctx) + if err != nil { + return err + } + for libName, lib := range libraries { + libPath := path.Join(dir, f.opt.Enc.ToStandardName(libName)) + if err := addEntry(fs.NewDir(libPath, f.startTime)); err != nil { + return err + } + albums, err := lib.GetAlbums(ctx) + if err != nil { + return err + } + if err := collectAlbumJobs(albums, libName, libPath); err != nil { + return err + } + } + + case strings.HasPrefix(dirID, "lib:"): + // Library level - emit and collect all albums + libraryName := strings.TrimPrefix(dirID, "lib:") + libraries, err := ps.GetLibraries(ctx) + if err != nil { + return err + } + library, exists := libraries[libraryName] + if !exists { + return fs.ErrorDirNotFound + } + albums, err := library.GetAlbums(ctx) + if err != nil { + return err + } + if err := collectAlbumJobs(albums, libraryName, dir); err != nil { + return err + } + + case strings.HasPrefix(dirID, "album:"): + // Album or folder level + libraryName, albumPath, ok := parseAlbumDirID(dirID) + if !ok { + return fs.ErrorDirNotFound + } + album, err := f.resolveAlbum(ctx, ps, libraryName, albumPath) + if err != nil { + return err + } + if album.IsFolder { + if err := collectAlbumJobs(album.Children, libraryName, dir); err != nil { + return err + } + } else { + // Single album - just list its photos + jobs = append(jobs, listRAlbumJob{album: album, zone: libraryName, dirPath: dir}) + } + + default: + return fs.ErrorDirNotFound + } + + if len(jobs) == 0 { + return helper.Flush() + } + + // Fan out album photo listing across goroutines + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + jobCh := make(chan listRAlbumJob, len(jobs)) + for _, job := range jobs { + jobCh <- job + } + close(jobCh) + + workers := fs.GetConfig(ctx).Checkers + if len(jobs) < workers { + workers = len(jobs) + } + errs := make(chan error, workers) + + for range workers { + go func() { + for job := range jobCh { + photos, err := job.album.GetPhotos(ctx) + if err != nil { + errs <- err + cancel() + return + } + for _, photo := range photos { + if photo.Filename == "" { + continue + } + remotePath := path.Join(job.dirPath, f.opt.Enc.ToStandardName(photo.Filename)) + o := f.newPhotosObject(remotePath, photo, job.zone) + if err := addEntry(o); err != nil { + errs <- err + cancel() + return + } + } + } + errs <- nil + }() + } + + for range workers { + if e := <-errs; e != nil && err == nil { + err = e + } + } + if err != nil { + return err + } + + return helper.Flush() +} + +func notifyAlbumTree(albums map[string]*api.Album, base string, notifyFunc func(string, fs.EntryType)) { + for name, album := range albums { + albumPath := name + if base != "" { + albumPath = path.Join(base, name) + } + notifyFunc(albumPath, fs.EntryDirectory) + if album.IsFolder { + notifyAlbumTree(album.Children, albumPath, notifyFunc) + } + } +} + +func (f *PhotosFs) notifyZoneChange(ctx context.Context, ps *api.PhotosService, zone string, notifyFunc func(string, fs.EntryType)) { + if f.root == "" { + notifyFunc(zone, fs.EntryDirectory) + return + } + + libraryName, albumPath, _ := strings.Cut(f.root, "/") + if libraryName != zone { + return + } + + // Always invalidate the mounted root when its backing zone changes + notifyFunc("", fs.EntryDirectory) + + libraries, err := ps.GetLibraries(ctx) + if err != nil { + return + } + library, ok := libraries[zone] + if !ok { + return + } + albums, err := library.GetAlbums(ctx) + if err != nil { + return + } + if albumPath == "" { + notifyAlbumTree(albums, "", notifyFunc) + return + } + + album, err := resolveAlbumPath(albums, albumPath) + if err != nil { + return + } + if album.IsFolder { + notifyAlbumTree(album.Children, "", notifyFunc) + } +} + +// ChangeNotify polls for changes and notifies the VFS when directories are modified +func (f *PhotosFs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) { + go func() { + var ticker *time.Ticker + var tickerC <-chan time.Time + for { + select { + case pollInterval, ok := <-pollIntervalChan: + if !ok { + if ticker != nil { + ticker.Stop() + } + return + } + if ticker != nil { + ticker.Stop() + tickerC = nil + } + if pollInterval > 0 { + ticker = time.NewTicker(pollInterval) + tickerC = ticker.C + } + case <-tickerC: + ps, err := f.photosService(ctx) + if err != nil { + fs.Debugf(f, "ChangeNotify: failed to get photos service: %v", err) + continue + } + changedZones := ps.PollForChanges(ctx) + for _, zone := range changedZones { + f.notifyZoneChange(ctx, ps, zone, notifyFunc) + } + case <-ctx.Done(): + if ticker != nil { + ticker.Stop() + } + return + } + } + }() +} + +// Disconnect clears authentication state and removes disk caches +func (f *PhotosFs) Disconnect(ctx context.Context) error { + return disconnectClient(f.m, f.icloud) +} + +// Check interfaces are satisfied +var ( + _ fs.Fs = (*PhotosFs)(nil) + _ fs.Disconnecter = (*PhotosFs)(nil) + _ fs.ListRer = (*PhotosFs)(nil) + _ fs.ChangeNotifier = (*PhotosFs)(nil) + _ fs.DirCacheFlusher = (*PhotosFs)(nil) + _ fs.Object = (*PhotosObject)(nil) + _ fs.Metadataer = (*PhotosObject)(nil) +) diff --git a/backend/iclouddrive/icloudphotos_test.go b/backend/iclouddrive/icloudphotos_test.go new file mode 100644 index 000000000..1aa32146d --- /dev/null +++ b/backend/iclouddrive/icloudphotos_test.go @@ -0,0 +1,516 @@ +//go:build !plan9 && !solaris + +package iclouddrive + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/rclone/rclone/backend/iclouddrive/api" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/dircache" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestPhotosFs() *PhotosFs { + return &PhotosFs{ + name: "test-icp", + root: "", + opt: Options{}, + } +} + +func TestSmartAlbumDefinitions(t *testing.T) { + expected := map[string]struct { + ObjectType string + ListType string + }{ + "All Photos": {"CPLAssetByAssetDateWithoutHiddenOrDeleted", "CPLAssetAndMasterByAssetDateWithoutHiddenOrDeleted"}, + "Time-lapse": {"CPLAssetInSmartAlbumByAssetDate:Timelapse", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Videos": {"CPLAssetInSmartAlbumByAssetDate:Video", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Slo-mo": {"CPLAssetInSmartAlbumByAssetDate:Slomo", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Bursts": {"CPLAssetBurstStackAssetByAssetDate", "CPLBurstStackAssetAndMasterByAssetDate"}, + "Favorites": {"CPLAssetInSmartAlbumByAssetDate:Favorite", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Panoramas": {"CPLAssetInSmartAlbumByAssetDate:Panorama", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Screenshots": {"CPLAssetInSmartAlbumByAssetDate:Screenshot", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Live": {"CPLAssetInSmartAlbumByAssetDate:Live", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Recently Deleted": {"CPLAssetDeletedByExpungedDate", "CPLAssetAndMasterDeletedByExpungedDate"}, + "Hidden": {"CPLAssetHiddenByAssetDate", "CPLAssetAndMasterHiddenByAssetDate"}, + "Portrait": {"CPLAssetInSmartAlbumByAssetDate:Depth", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Long Exposure": {"CPLAssetInSmartAlbumByAssetDate:Exposure", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + "Animated": {"CPLAssetInSmartAlbumByAssetDate:Animated", "CPLAssetAndMasterInSmartAlbumByAssetDate"}, + } + + for name, exp := range expected { + t.Run(name, func(t *testing.T) { + album, exists := api.SmartAlbums[name] + require.True(t, exists, "smart album %q must be defined", name) + assert.Equal(t, exp.ObjectType, album.ObjectType) + assert.Equal(t, exp.ListType, album.ListType) + }) + } + + assert.Equal(t, len(expected), len(api.SmartAlbums)) +} + +func TestSmartAlbumFilters(t *testing.T) { + filtered := map[string]string{ + "Time-lapse": "TIMELAPSE", + "Videos": "VIDEO", + "Slo-mo": "SLOMO", + "Favorites": "FAVORITE", + "Panoramas": "PANORAMA", + "Screenshots": "SCREENSHOT", + "Live": "LIVE", + "Portrait": "DEPTH", + "Long Exposure": "EXPOSURE", + "Animated": "ANIMATED", + } + + for name, filterVal := range filtered { + t.Run(name, func(t *testing.T) { + album := api.SmartAlbums[name] + require.NotEmpty(t, album.Filters, "album %q must have filters", name) + assert.Equal(t, "smartAlbum", album.Filters[0].FieldName) + assert.Equal(t, "EQUALS", album.Filters[0].Comparator) + fv, ok := album.Filters[0].FieldValue.(map[string]string) + require.True(t, ok, "filter value must be map[string]string") + assert.Equal(t, filterVal, fv["value"]) + }) + } + + unfiltered := []string{"All Photos", "Bursts", "Recently Deleted", "Hidden"} + for _, name := range unfiltered { + t.Run(name+"_no_filter", func(t *testing.T) { + album := api.SmartAlbums[name] + assert.Empty(t, album.Filters, "album %q must not have filters", name) + }) + } +} + +func TestParseAlbumDirID(t *testing.T) { + lib, album, ok := parseAlbumDirID("album:PrimarySync:Favorites") + assert.True(t, ok) + assert.Equal(t, "PrimarySync", lib) + assert.Equal(t, "Favorites", album) + + // Colon in album name + lib, album, ok = parseAlbumDirID("album:PrimarySync:My:Album") + assert.True(t, ok) + assert.Equal(t, "PrimarySync", lib) + assert.Equal(t, "My:Album", album) + + // Missing album + _, _, ok = parseAlbumDirID("album:PrimarySync") + assert.False(t, ok) + + // Empty + _, _, ok = parseAlbumDirID("album:") + assert.False(t, ok) +} + +func TestPhotosObject_Metadata(t *testing.T) { + o := &PhotosObject{ + fs: &PhotosFs{name: "test"}, + remote: "PrimarySync/All Photos/IMG_0001.HEIC", + size: 4200000, + modTime: time.Date(2025, 6, 15, 14, 30, 0, 0, time.UTC), + masterID: "master-001", + zone: "PrimarySync", + width: 4032, + height: 3024, + addedDate: 1718459400000, // 2024-06-15T13:50:00Z in millis + isFavorite: true, + isHidden: false, + } + + metadata, err := o.Metadata(context.Background()) + require.NoError(t, err) + assert.Equal(t, "4032", metadata["width"]) + assert.Equal(t, "3024", metadata["height"]) + assert.Equal(t, "2024-06-15T13:50:00Z", metadata["added-time"]) + assert.Equal(t, "true", metadata["favorite"]) + assert.Equal(t, "false", metadata["hidden"]) +} + +func TestPhotosObject_MetadataZeroDimensions(t *testing.T) { + // Live Photo .MOV companion - no width/height, not favorite + o := &PhotosObject{ + fs: &PhotosFs{name: "test"}, + remote: "PrimarySync/Live/IMG_3031.MOV", + } + + metadata, err := o.Metadata(context.Background()) + require.NoError(t, err) + _, hasWidth := metadata["width"] + _, hasHeight := metadata["height"] + assert.False(t, hasWidth, "zero width should be omitted") + assert.False(t, hasHeight, "zero height should be omitted") + assert.Equal(t, "false", metadata["favorite"]) + assert.Equal(t, "false", metadata["hidden"]) +} + +// newTestPhotosService builds a PhotosService with pre-populated libraries and albums +// for testing resolveAlbum and FindLeaf without HTTP calls +func newTestPhotosService() *api.PhotosService { + return api.NewTestPhotosService(map[string]map[string]*api.Album{ + "PrimarySync": { + "All Photos": {Name: "All Photos", ObjectType: "CPLAssetByAssetDateWithoutHiddenOrDeleted"}, + "Videos": {Name: "Videos", ObjectType: "CPLAssetInSmartAlbumByAssetDate:Video"}, + "UserAlbum": {Name: "UserAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec1", RecordName: "rec1"}, + "Folder": { + Name: "Folder", RecordName: "folder1", IsFolder: true, + Children: map[string]*api.Album{ + "ChildAlbum": {Name: "ChildAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec2", RecordName: "rec2"}, + "NestedFolder": { + Name: "NestedFolder", RecordName: "folder2", IsFolder: true, + Children: map[string]*api.Album{ + "LeafAlbum": {Name: "LeafAlbum", ObjectType: "CPLContainerRelationNotDeletedByAssetDate:rec3", RecordName: "rec3"}, + }, + }, + }, + }, + }, + }) +} + +func TestResolveAlbum(t *testing.T) { + f := newTestPhotosFs() + f.photos = newTestPhotosService() + ctx := context.Background() + + t.Run("simple album", func(t *testing.T) { + album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos") + require.NoError(t, err) + assert.Equal(t, "Videos", album.Name) + }) + + t.Run("nested folder child", func(t *testing.T) { + album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/ChildAlbum") + require.NoError(t, err) + assert.Equal(t, "ChildAlbum", album.Name) + }) + + t.Run("two-level nesting", func(t *testing.T) { + album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder/NestedFolder/LeafAlbum") + require.NoError(t, err) + assert.Equal(t, "LeafAlbum", album.Name) + }) + + t.Run("folder itself", func(t *testing.T) { + album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder") + require.NoError(t, err) + assert.True(t, album.IsFolder) + }) + + t.Run("missing album", func(t *testing.T) { + _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "NoSuchAlbum") + assert.ErrorIs(t, err, fs.ErrorDirNotFound) + }) + + t.Run("missing library", func(t *testing.T) { + _, err := f.resolveAlbum(ctx, f.photos, "NoSuchLib", "Videos") + assert.ErrorIs(t, err, fs.ErrorDirNotFound) + }) + + t.Run("traverse into non-folder", func(t *testing.T) { + _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Videos/SubAlbum") + assert.ErrorIs(t, err, fs.ErrorDirNotFound) + }) +} + +func TestFindLeaf(t *testing.T) { + f := newTestPhotosFs() + f.photos = newTestPhotosService() + ctx := context.Background() + + t.Run("root to library", func(t *testing.T) { + id, found, err := f.FindLeaf(ctx, rootID, "PrimarySync") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "lib:PrimarySync", id) + }) + + t.Run("root to missing library", func(t *testing.T) { + _, found, err := f.FindLeaf(ctx, rootID, "NoSuchLib") + require.NoError(t, err) + assert.False(t, found) + }) + + t.Run("library to album", func(t *testing.T) { + id, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "Videos") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "album:PrimarySync:Videos", id) + }) + + t.Run("library to missing album", func(t *testing.T) { + _, found, err := f.FindLeaf(ctx, "lib:PrimarySync", "NoSuchAlbum") + require.NoError(t, err) + assert.False(t, found) + }) + + t.Run("missing library returns ErrorDirNotFound", func(t *testing.T) { + _, _, err := f.FindLeaf(ctx, "lib:NoSuchLib", "Videos") + assert.ErrorIs(t, err, fs.ErrorDirNotFound) + }) + + t.Run("folder to child album", func(t *testing.T) { + id, found, err := f.FindLeaf(ctx, "album:PrimarySync:Folder", "ChildAlbum") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "album:PrimarySync:Folder/ChildAlbum", id) + }) + + t.Run("non-folder album returns not found", func(t *testing.T) { + _, found, err := f.FindLeaf(ctx, "album:PrimarySync:Videos", "SubAlbum") + require.NoError(t, err) + assert.False(t, found) + }) + + t.Run("unknown prefix returns not found", func(t *testing.T) { + _, found, err := f.FindLeaf(ctx, "unknown:prefix", "leaf") + require.NoError(t, err) + assert.False(t, found) + }) +} + +func TestResolveAlbum_EmptyPath(t *testing.T) { + f := newTestPhotosFs() + f.photos = newTestPhotosService() + ctx := context.Background() + + // parseAlbumDirID("album:PrimarySync:") returns ("PrimarySync", "", true) + // This empty album path feeds into resolveAlbum + _, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "") + assert.ErrorIs(t, err, fs.ErrorDirNotFound, "empty album path should return ErrorDirNotFound") +} + +func TestNewObject_ErrorPaths(t *testing.T) { + f := newTestPhotosFs() + f.photos = newTestPhotosService() + ctx := context.Background() + + // Pre-populate photoCache on the "Videos" album so GetPhotoByName works + // without HTTP. Access via the test PhotosService's internal structure + libs, _ := f.photos.GetLibraries(ctx) + albums, _ := libs["PrimarySync"].GetAlbums(ctx) + videos := albums["Videos"] + videos.SetTestPhotoCache(map[string]*api.Photo{ + "existing.mp4": {ID: "m1", Filename: "existing.mp4", Size: 1024, ResourceKey: "resOriginalRes"}, + }) + + // NewObject depends on dircache.FindDir which requires FindRoot + HTTP + // Test the reachable paths through resolveAlbum and GetPhotoByName + + // Test resolveAlbum → folder → GetPhotoByName on folder (should fail) + album, err := f.resolveAlbum(ctx, f.photos, "PrimarySync", "Folder") + require.NoError(t, err) + assert.True(t, album.IsFolder, "Folder should be a folder") + // NewObject would return ErrorObjectNotFound for a folder path + + // Test GetPhotoByName on album with populated cache + photo, err := videos.GetPhotoByName(ctx, "existing.mp4") + require.NoError(t, err) + assert.Equal(t, "m1", photo.ID) + + // Test GetPhotoByName cache miss + _, err = videos.GetPhotoByName(ctx, "nonexistent.mp4") + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent.mp4") +} + +func TestParseAlbumDirID_Exhaustive(t *testing.T) { + tests := []struct { + input string + lib, album string + ok bool + }{ + {"album:PrimarySync:Videos", "PrimarySync", "Videos", true}, + {"album:PrimarySync:Folder/Child", "PrimarySync", "Folder/Child", true}, + {"album:PrimarySync:Name:With:Colons", "PrimarySync", "Name:With:Colons", true}, + {"album:PrimarySync:", "PrimarySync", "", true}, + {"album:", "", "", false}, + {"notalbum:foo:bar", "notalbum", "foo:bar", true}, // strips "album:" prefix literally, "notalbum:" stays + {"", "", "", false}, + } + + for _, tt := range tests { + lib, album, ok := parseAlbumDirID(tt.input) + if tt.ok { + assert.True(t, ok, "input=%q", tt.input) + assert.Equal(t, tt.lib, lib, "input=%q", tt.input) + assert.Equal(t, tt.album, album, "input=%q", tt.input) + } else { + assert.False(t, ok, "input=%q", tt.input) + } + } +} + +// setEmptyPhotoCaches recursively sets empty photo caches on all leaf albums +// so GetPhotos returns without HTTP (service=nil test fast path) +func setEmptyPhotoCaches(albums map[string]*api.Album) { + for _, album := range albums { + if album.IsFolder { + setEmptyPhotoCaches(album.Children) + } else { + album.SetTestPhotoCache(map[string]*api.Photo{}) + } + } +} + +func TestListR_NestedFolderRecursion(t *testing.T) { + f := newTestPhotosFs() + f.photos = newTestPhotosService() + f.dirCache = dircache.New("", rootID, f) + f.features = (&fs.Features{}).Fill(context.Background(), f) + f.startTime = time.Now() + ctx := context.Background() + + err := f.dirCache.FindRoot(ctx, false) + require.NoError(t, err) + + // Pre-populate empty photo caches on all leaf albums + libs, err := f.photos.GetLibraries(ctx) + require.NoError(t, err) + for _, lib := range libs { + albums, err := lib.GetAlbums(ctx) + require.NoError(t, err) + setEmptyPhotoCaches(albums) + } + + var dirs []string + err = f.ListR(ctx, "", func(entries fs.DirEntries) error { + for _, entry := range entries { + if _, ok := entry.(fs.Directory); ok { + dirs = append(dirs, entry.Remote()) + } + } + return nil + }) + require.NoError(t, err) + sort.Strings(dirs) + + // All directories must be present, including deeply nested ones + assert.Contains(t, dirs, "PrimarySync") + assert.Contains(t, dirs, "PrimarySync/All Photos") + assert.Contains(t, dirs, "PrimarySync/Videos") + assert.Contains(t, dirs, "PrimarySync/Folder") + assert.Contains(t, dirs, "PrimarySync/Folder/ChildAlbum") + assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder") + assert.Contains(t, dirs, "PrimarySync/Folder/NestedFolder/LeafAlbum") +} + +func TestListR_FolderLevel(t *testing.T) { + f := &PhotosFs{ + name: "test", + root: "PrimarySync/Folder", + opt: Options{}, + startTime: time.Now(), + } + f.features = (&fs.Features{}).Fill(context.Background(), f) + f.photos = newTestPhotosService() + f.dirCache = dircache.New("PrimarySync/Folder", rootID, f) + ctx := context.Background() + + err := f.dirCache.FindRoot(ctx, false) + require.NoError(t, err) + + libs, _ := f.photos.GetLibraries(ctx) + for _, lib := range libs { + albums, _ := lib.GetAlbums(ctx) + setEmptyPhotoCaches(albums) + } + + var dirs []string + err = f.ListR(ctx, "", func(entries fs.DirEntries) error { + for _, entry := range entries { + if _, ok := entry.(fs.Directory); ok { + dirs = append(dirs, entry.Remote()) + } + } + return nil + }) + require.NoError(t, err) + + // When rooted at a folder, should see children recursively + assert.Contains(t, dirs, "ChildAlbum") + assert.Contains(t, dirs, "NestedFolder") + assert.Contains(t, dirs, "NestedFolder/LeafAlbum") +} + +func TestNotifyZoneChange(t *testing.T) { + ctx := context.Background() + ps := newTestPhotosService() + + tests := []struct { + name string + root string + zone string + want []string + }{ + { + name: "top level root", + root: "", + zone: "PrimarySync", + want: []string{"PrimarySync"}, + }, + { + name: "library root", + root: "PrimarySync", + zone: "PrimarySync", + want: []string{"", "All Photos", "Folder", "Folder/ChildAlbum", "Folder/NestedFolder", "Folder/NestedFolder/LeafAlbum", "UserAlbum", "Videos"}, + }, + { + name: "folder root", + root: "PrimarySync/Folder", + zone: "PrimarySync", + want: []string{"", "ChildAlbum", "NestedFolder", "NestedFolder/LeafAlbum"}, + }, + { + name: "nested folder root", + root: "PrimarySync/Folder/NestedFolder", + zone: "PrimarySync", + want: []string{"", "LeafAlbum"}, + }, + { + name: "leaf album root", + root: "PrimarySync/Folder/NestedFolder/LeafAlbum", + zone: "PrimarySync", + want: []string{""}, + }, + { + name: "missing nested album still invalidates root", + root: "PrimarySync/Folder/NoSuchAlbum", + zone: "PrimarySync", + want: []string{""}, + }, + { + name: "different zone ignored", + root: "PrimarySync/Folder/NestedFolder", + zone: "SharedSync", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newTestPhotosFs() + f.root = tt.root + + var got []string + f.notifyZoneChange(ctx, ps, tt.zone, func(remote string, entryType fs.EntryType) { + assert.Equal(t, fs.EntryDirectory, entryType) + got = append(got, remote) + }) + + sort.Strings(got) + want := append([]string(nil), tt.want...) + sort.Strings(want) + assert.Equal(t, want, got) + }) + } +} diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.md index af18796fd..22c488757 100644 --- a/docs/content/iclouddrive.md +++ b/docs/content/iclouddrive.md @@ -1,20 +1,25 @@ --- -title: "iCloud Drive" -description: "Rclone docs for iCloud Drive" +title: "iCloud Drive and Photos" +description: "Rclone docs for iCloud Drive and Photos" versionIntroduced: "v1.69" --- -# iCloud Drive +# {{< icon "fa fa-cloud" >}} iCloud Drive and Photos ## Configuration -The initial setup for an iCloud Drive backend involves getting a trust token/session. -This can be done by simply using the regular iCloud password, and accepting the code -prompt on another iCloud connected device. +The initial setup for an iCloud backend involves getting a trust token/session. +This uses your regular Apple ID password plus 2FA, either from a trusted +device prompt or an SMS code sent to a trusted phone number. **IMPORTANT**: App-specific passwords are not accepted. Only use your regular Apple ID password and 2FA. +This backend serves two Apple services: + +- `drive` - iCloud Drive (default) +- `photos` - iCloud Photos + `rclone config` walks you through the token creation. The trust token is valid for 30 days. After which you will have to reauthenticate with `rclone reconnect` or `rclone config`. @@ -30,10 +35,12 @@ The authentication flow is: 1. rclone initiates a session with Apple's identity service 2. An SRP key exchange takes place (your password is used locally to derive a key) -3. Apple sends a 2FA prompt to your trusted devices +3. Apple sends a 2FA prompt to your trusted devices, or lets you request an + SMS code 4. After you enter the 2FA code, rclone receives a trust token for future sessions -Here is an example of how to make a remote called `iclouddrive`. First run: +Here is an example of how to make a Photos remote called `icloudphotos`. +For iCloud Drive, leave `service` at its default `drive` value. First run: ```console rclone config @@ -47,7 +54,7 @@ n) New remote s) Set configuration password q) Quit config n/s/q> n -name> iclouddrive +name> icloudphotos Option Storage. Type of storage to configure. Choose a number from below, or type in your own value. @@ -56,6 +63,15 @@ XX / iCloud Drive \ (iclouddrive) [snip] Storage> iclouddrive +Option service. +iCloud service to use. +Choose a number from below, or type in your own value of type string. +Press Enter for the default (drive). + 1 / iCloud Drive + \ (drive) + 2 / iCloud Photos + \ (photos) +service> 2 Option apple_id. Apple ID. Enter a value. @@ -75,13 +91,14 @@ y) Yes n) No (default) y/n> n Option config_2fa. -Two-factor authentication: please enter your 2FA code +Two-factor authentication: enter your 2FA code or type 'sms' for a text message Enter a value. config_2fa> 2FACODE Remote config -------------------- -[iclouddrive] +[icloudphotos] - type: iclouddrive +- service: photos - apple_id: APPLEID - password: *** ENCRYPTED *** - cookies: **************************** @@ -93,30 +110,124 @@ d) Delete this remote y/e/d> y ``` +## iCloud Photos + +The iCloud Drive backend also supports accessing iCloud Photos by setting the +`service` option to `photos`: + +```console +rclone lsd iclouddrive: --iclouddrive-service photos +``` + +This presents a read-only hierarchy rooted at photo libraries: + +- **Level 1**: Photo libraries — your personal library (`PrimarySync`) and + any Shared Photo Library (`SharedSync-XXXX`) +- **Level 2+**: Albums and folders within a library, nested recursively as in + Apple Photos +- **Leaf level**: Photos/videos within an album, including Live Photo `.MOV` + companions + +Examples: + +```console +# List libraries +rclone lsd iclouddrive: --iclouddrive-service photos + +# List albums in your primary library +rclone lsd iclouddrive:PrimarySync/ --iclouddrive-service photos + +# List photos in an album +rclone ls iclouddrive:PrimarySync/All\ Photos/ --iclouddrive-service photos + +# Download a photo +rclone copy iclouddrive:PrimarySync/Favorites/IMG_0001.HEIC /tmp/ --iclouddrive-service photos +``` + +You can either: + +- set `service = photos` in `rclone config` for a dedicated Photos remote +- keep `service = drive` and pass `--iclouddrive-service photos` when needed + +### Metadata + +With `--metadata`, Photos entries expose these read-only metadata keys: + +- `width` +- `height` +- `added-time` +- `favorite` +- `hidden` + +These metadata keys are only available when `service = photos`. + +### Caching + +iCloud Photos caches album listings to disk for fast subsequent access. +On the first run, listing a large album uses parallel `startRank` +partitions to fetch pages concurrently. After that, a lightweight change +check (~200ms) determines whether the cache is still valid. + +Cache location: `~/.cache/rclone/iclouddrive-photos///` + +To clear the cache: delete that directory or run `rclone config reconnect`. + +### FUSE mounts + +For mounting iCloud Photos via `rclone mount`, the following flags +are recommended: + + rclone mount remote: /mnt/photos \ + --iclouddrive-service photos \ + --vfs-refresh \ + --dir-cache-time 1h \ + --vfs-cache-mode full \ + --attr-timeout 1m \ + --read-only + +- `--vfs-refresh` pre-warms directory caches in the background on mount + start so that albums are ready when you browse them +- `--dir-cache-time 1h` extends the in-memory cache lifetime beyond the + default 5 minutes (change detection is fast, so this is safe) +- `--vfs-cache-mode full` caches downloaded photos and videos to local + disk for fast repeated access +- `--attr-timeout 1m` reduces kernel attribute lookups (safe because + the backend is read-only) +- `--read-only` prevents confusing write errors + +The first listing of a very large album (e.g. 75,000 items in "All +Photos") can take several minutes due to API pagination limits. This +happens once — subsequent listings use the disk cache. + +### Limitations + +iCloud Photos is read-only. Upload, delete, rename, and move operations +are not supported. + ## Advanced Data Protection -ADP is currently unsupported and need to be disabled +Advanced Data Protection is supported. On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web' -must be ON, and 'Advanced Data Protection' OFF. +must be ON. + +If ADP is enabled on your account, rclone requests PCS cookies after 2FA. +Apple may require approval on a trusted device before those cookies are issued. ## Troubleshooting -### Missing PCS cookies from the request +### PCS cookie errors with ADP -This means you have Advanced Data Protection (ADP) turned on. This is not supported -at the moment. If you want to use rclone you will have to turn it off. See above -for how to turn it off. +If you see `Missing PCS cookies from the request` or a `requestPCS:` error, +the ADP approval flow did not complete successfully. -You will need to clear the `cookies` and the `trust_token` fields in the config. -Or you can delete the remote config and start again. +Check that 'Access iCloud Data on the Web' is enabled and approve any prompt +on your trusted device. -You should then run `rclone reconnect remote:`. +Then run `rclone reconnect remote:`. -Note that changing the ADP setting may not take effect immediately - you may -need to wait a few hours or a day before you can get rclone to work - keep -clearing the config entry and running `rclone reconnect remote:` until rclone -functions properly. +If the remote still has stale auth state, clear the `cookies` and +`trust_token` fields in the config, or delete and recreate the remote. ### Standard options