mirror of https://github.com/immich-app/immich.git
background upload plugin
add schemas sync variants formatting initial implementation use existing db, wip move to separate folder fix table definitions wip wiring it up repository patternpull/23700/head
mertalev
parent
41f013387f
commit
92bc22620b
@ -0,0 +1,279 @@
|
||||
import SQLiteData
|
||||
|
||||
protocol TaskProtocol {
|
||||
func getTaskIds(status: TaskStatus) async throws -> [Int64]
|
||||
func getBackupCandidates() async throws -> [LocalAssetCandidate]
|
||||
func getBackupCandidates(ids: [String]) async throws -> [LocalAssetCandidate]
|
||||
func getDownloadTasks() async throws -> [LocalAssetDownloadData]
|
||||
func getUploadTasks() async throws -> [LocalAssetUploadData]
|
||||
func markOrphansPending(ids: [Int64]) async throws
|
||||
func markDownloadQueued(taskId: Int64, isLivePhoto: Bool, filePath: URL) async throws
|
||||
func markUploadQueued(taskId: Int64) async throws
|
||||
func markDownloadComplete(taskId: Int64, localId: String, hash: String?) async throws -> TaskStatus
|
||||
func markUploadSuccess(taskId: Int64, livePhotoVideoId: String?) async throws
|
||||
func retryOrFail(taskId: Int64, code: UploadErrorCode, status: TaskStatus) async throws
|
||||
func enqueue(assets: [LocalAssetCandidate], imagePriority: Float, videoPriority: Float) async throws
|
||||
func enqueue(files: [String]) async throws
|
||||
func resolveError(code: UploadErrorCode) async throws
|
||||
func getFilename(taskId: Int64) async throws -> String?
|
||||
}
|
||||
|
||||
final class TaskRepository: TaskProtocol {
|
||||
private let db: DatabasePool
|
||||
|
||||
init(db: DatabasePool) {
|
||||
self.db = db
|
||||
}
|
||||
|
||||
func getTaskIds(status: TaskStatus) async throws -> [Int64] {
|
||||
return try await db.read { conn in
|
||||
try UploadTask.select(\.id).where { $0.status.eq(status) }.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getBackupCandidates() async throws -> [LocalAssetCandidate] {
|
||||
return try await db.read { conn in
|
||||
return try LocalAsset.backupCandidates.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getBackupCandidates(ids: [String]) async throws -> [LocalAssetCandidate] {
|
||||
return try await db.read { conn in
|
||||
return try LocalAsset.backupCandidates.where { $0.id.in(ids) }.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getDownloadTasks() async throws -> [LocalAssetDownloadData] {
|
||||
return try await db.read({ conn in
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, _ in task.canRetry && task.noFatalError && LocalAsset.withChecksum.exists() }
|
||||
.select { task, asset in
|
||||
LocalAssetDownloadData.Columns(
|
||||
checksum: asset.checksum,
|
||||
createdAt: asset.createdAt,
|
||||
filename: asset.name,
|
||||
livePhotoVideoId: task.livePhotoVideoId,
|
||||
localId: asset.id,
|
||||
taskId: task.id,
|
||||
updatedAt: asset.updatedAt
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { _, _ in UploadTaskStat.availableDownloadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
}
|
||||
|
||||
func getUploadTasks() async throws -> [LocalAssetUploadData] {
|
||||
return try await db.read({ conn in
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, _ in task.canRetry && task.noFatalError && LocalAsset.withChecksum.exists() }
|
||||
.select { task, asset in
|
||||
LocalAssetUploadData.Columns(
|
||||
filename: asset.name,
|
||||
filePath: task.filePath.unwrapped,
|
||||
priority: task.priority,
|
||||
taskId: task.id,
|
||||
type: asset.type
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { task, _ in UploadTaskStat.availableUploadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
}
|
||||
|
||||
func markOrphansPending(ids: [Int64]) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.filePath = nil
|
||||
$0.status = .downloadPending
|
||||
}
|
||||
.where { row in row.status.in([TaskStatus.downloadQueued, TaskStatus.uploadPending]) || row.id.in(ids) }
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markDownloadQueued(taskId: Int64, isLivePhoto: Bool, filePath: URL) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.status = .downloadQueued
|
||||
$0.isLivePhoto = isLivePhoto
|
||||
$0.filePath = filePath
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markUploadQueued(taskId: Int64) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
row.status = .uploadQueued
|
||||
row.filePath = nil
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markDownloadComplete(taskId: Int64, localId: String, hash: String?) async throws -> TaskStatus {
|
||||
return try await db.write { conn in
|
||||
if let hash {
|
||||
try LocalAsset.update { $0.checksum = hash }.where { $0.id.eq(localId) }.execute(conn)
|
||||
}
|
||||
let status =
|
||||
if let hash, try RemoteAsset.select(\.rowid).where({ $0.checksum.eq(hash) }).fetchOne(conn) != nil {
|
||||
TaskStatus.uploadSkipped
|
||||
} else {
|
||||
TaskStatus.uploadPending
|
||||
}
|
||||
try UploadTask.update { $0.status = status }.where { $0.id.eq(taskId) }.execute(conn)
|
||||
return status
|
||||
}
|
||||
}
|
||||
|
||||
func markUploadSuccess(taskId: Int64, livePhotoVideoId: String?) async throws {
|
||||
try await db.write { conn in
|
||||
let task =
|
||||
try UploadTask
|
||||
.update { $0.status = .uploadComplete }
|
||||
.where { $0.id.eq(taskId) }
|
||||
.returning(\.self)
|
||||
.fetchOne(conn)
|
||||
guard let task, let localId = task.localId, let isLivePhoto = task.isLivePhoto, isLivePhoto,
|
||||
task.livePhotoVideoId == nil
|
||||
else { return }
|
||||
try UploadTask.insert {
|
||||
UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: true,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: livePhotoVideoId,
|
||||
localId: localId,
|
||||
method: .multipart,
|
||||
priority: 0.7,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func retryOrFail(taskId: Int64, code: UploadErrorCode, status: TaskStatus) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
let retryOffset =
|
||||
switch code {
|
||||
case .iCloudThrottled, .iCloudRateLimit, .notEnoughSpace: 3000
|
||||
default: 0
|
||||
}
|
||||
row.status = Case()
|
||||
.when(row.localId.is(nil) && row.attempts.lte(TaskConfig.maxRetries), then: TaskStatus.uploadPending)
|
||||
.when(row.attempts.lte(TaskConfig.maxRetries), then: TaskStatus.downloadPending)
|
||||
.else(status)
|
||||
row.attempts += 1
|
||||
row.lastError = code
|
||||
row.retryAfter = #sql("unixepoch('now') + (\(4 << row.attempts)) + \(retryOffset)")
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(assets: [LocalAssetCandidate], imagePriority: Float, videoPriority: Float) async throws {
|
||||
try await db.write { conn in
|
||||
var draft = draftStub
|
||||
for candidate in assets {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? imagePriority : videoPriority
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(files: [String]) async throws {
|
||||
try await db.write { conn in
|
||||
var draft = draftStub
|
||||
draft.priority = 1.0
|
||||
draft.status = .uploadPending
|
||||
for file in files {
|
||||
draft.filePath = URL(fileURLWithPath: file, isDirectory: false)
|
||||
try UploadTask.insert { draft }.execute(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resolveError(code: UploadErrorCode) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { $0.lastError = nil }.where { $0.lastError.unwrapped.eq(code) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getFilename(taskId: Int64) async throws -> String? {
|
||||
try await db.read { conn in
|
||||
try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }.select(\.1.name).fetchOne(conn)
|
||||
}
|
||||
}
|
||||
|
||||
private var draftStub: UploadTask.Draft {
|
||||
.init(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: nil,
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extension UploadTask.TableColumns {
|
||||
var noFatalError: some QueryExpression<Bool> { lastError.is(nil) || !lastError.unwrapped.in(UploadErrorCode.fatal) }
|
||||
var canRetry: some QueryExpression<Bool> {
|
||||
attempts.lte(TaskConfig.maxRetries) && (retryAfter.is(nil) || retryAfter.unwrapped <= Date().unixTime)
|
||||
}
|
||||
}
|
||||
|
||||
extension LocalAlbum {
|
||||
static let selected = Self.where { $0.backupSelection.eq(BackupSelection.selected) }
|
||||
static let excluded = Self.where { $0.backupSelection.eq(BackupSelection.excluded) }
|
||||
}
|
||||
|
||||
extension LocalAlbumAsset {
|
||||
static let selected = Self.where {
|
||||
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.selected.select(\.id))
|
||||
}
|
||||
static let excluded = Self.where {
|
||||
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.excluded.select(\.id))
|
||||
}
|
||||
}
|
||||
|
||||
extension RemoteAsset {
|
||||
static let currentUser = Self.where { _ in
|
||||
ownerId.eq(Store.select(\.stringValue).where { $0.id.eq(StoreKey.currentUser.rawValue) }.unwrapped)
|
||||
}
|
||||
}
|
||||
|
||||
extension LocalAsset {
|
||||
static let withChecksum = Self.where { $0.checksum.isNot(nil) }
|
||||
static let shouldBackup = Self.where { _ in LocalAlbumAsset.selected.exists() && !LocalAlbumAsset.excluded.exists() }
|
||||
static let notBackedUp = Self.where { local in
|
||||
!RemoteAsset.currentUser.where { remote in local.checksum.eq(remote.checksum) }.exists()
|
||||
}
|
||||
static let backupCandidates = Self
|
||||
.shouldBackup
|
||||
.notBackedUp
|
||||
.where { local in !UploadTask.where { $0.localId.eq(local.id) }.exists() }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
import StructuredFieldValues
|
||||
|
||||
struct AssetData: StructuredFieldValue {
|
||||
static let structuredFieldType: StructuredFieldType = .dictionary
|
||||
|
||||
let deviceAssetId: String
|
||||
let deviceId: String
|
||||
let fileCreatedAt: String
|
||||
let fileModifiedAt: String
|
||||
let fileName: String
|
||||
let isFavorite: Bool
|
||||
let livePhotoVideoId: String?
|
||||
|
||||
static let boundary = "Boundary-\(UUID().uuidString)"
|
||||
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileCreatedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileModifiedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let livePhotoVideoIdField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let trueData = "true".data(using: .utf8)!
|
||||
static let falseData = "false".data(using: .utf8)!
|
||||
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
|
||||
static let contentType = "multipart/form-data; boundary=\(boundary)"
|
||||
|
||||
func multipart() -> (Data, Data) {
|
||||
var header = Data()
|
||||
header.append(Self.deviceAssetIdField)
|
||||
header.append(deviceAssetId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.deviceIdField)
|
||||
header.append(deviceId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileCreatedAtField)
|
||||
header.append(fileCreatedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileModifiedAtField)
|
||||
header.append(fileModifiedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.isFavoriteField)
|
||||
header.append(isFavorite ? Self.trueData : Self.falseData)
|
||||
|
||||
if let livePhotoVideoId {
|
||||
header.append(Self.livePhotoVideoIdField)
|
||||
header.append(livePhotoVideoId.data(using: .utf8)!)
|
||||
}
|
||||
header.append(
|
||||
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
)
|
||||
return (header, Self.footer)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue