sascha
/
immich
mirror of https://github.com/immich-app/immich.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: sync repository (#19581)

pull/19583/head
Jason Rasmussen 2025-06-27 13:47:06 +07:00 committed by GitHub
parent 6feca56da8
commit df76735f4a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 793 additions and 704 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/bin/sync-sql.ts
Unescape Escape View File

@ -15,6 +15,7 @@ import { repositories } from 'src/repositories';
import { AccessRepository } from 'src/repositories/access.repository'; import { AccessRepository } from 'src/repositories/access.repository';
import { ConfigRepository } from 'src/repositories/config.repository'; import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository'; import { LoggingRepository } from 'src/repositories/logging.repository';
import { SyncRepository } from 'src/repositories/sync.repository';
import { AuthService } from 'src/services/auth.service'; import { AuthService } from 'src/services/auth.service';
import { getKyselyConfig } from 'src/utils/database'; import { getKyselyConfig } from 'src/utils/database';
@ -111,7 +112,7 @@ class SqlGenerator {
data.push(...(await this.runTargets(instance, `${Repository.name}`))); data.push(...(await this.runTargets(instance, `${Repository.name}`)));
// nested repositories // nested repositories
if (Repository.name === AccessRepository.name) { if (Repository.name === AccessRepository.name || Repository.name === SyncRepository.name) {
for (const key of Object.keys(instance)) { for (const key of Object.keys(instance)) {
const subInstance = (instance as any)[key]; const subInstance = (instance as any)[key];
data.push(...(await this.runTargets(subInstance, `${Repository.name}.${key}`))); data.push(...(await this.runTargets(subInstance, `${Repository.name}.${key}`)));

15
server/src/queries/sync.checkpoint.repository.sql
Unescape Escape View File

@ -0,0 +1,15 @@
-- NOTE: This file is auto generated by ./sql-generator
-- SyncCheckpointRepository.getAll
select
"type",
"ack"
from
"session_sync_checkpoints"
where
"sessionId" = $1
-- SyncCheckpointRepository.deleteAll
delete from "session_sync_checkpoints"
where
"sessionId" = $1

752
server/src/queries/sync.repository.sql
Unescape Escape View File

@ -1,115 +1,55 @@
-- NOTE: This file is auto generated by ./sql-generator -- NOTE: This file is auto generated by ./sql-generator
-- SyncRepository.getCheckpoints -- SyncRepository.album.getCreatedAfter
select select
"type", "albumsId" as "id",
"ack" "createId"
from
"session_sync_checkpoints"
where
"sessionId" = $1
-- SyncRepository.deleteCheckpoints
delete from "session_sync_checkpoints"
where
"sessionId" = $1
-- SyncRepository.getUserUpserts
select
"id",
"name",
"email",
"deletedAt",
"updateId"
from
"users"
where
"updatedAt" < now() - interval '1 millisecond'
order by
"updateId" asc
-- SyncRepository.getUserDeletes
select
"id",
"userId"
from
"users_audit"
where
"deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.getPartnerUpserts
select
"sharedById",
"sharedWithId",
"inTimeline",
"updateId"
from from
"partners" "albums_shared_users_users"
where where
( "usersId" = $1
"sharedById" = $1 and "createId" >= $2
or "sharedWithId" = $2 and "createdAt" < now() - interval '1 millisecond'
)
and "updatedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "createId" asc
-- SyncRepository.getPartnerDeletes -- SyncRepository.album.getDeletes
select select
"id", "id",
"sharedById", "albumId"
"sharedWithId"
from from
"partners_audit" "albums_audit"
where where
( "userId" = $1
"sharedById" = $1
or "sharedWithId" = $2
)
and "deletedAt" < now() - interval '1 millisecond' and "deletedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "id" asc
-- SyncRepository.getAssetUpserts -- SyncRepository.album.getUpserts
select select distinct
"assets"."id", on ("albums"."id", "albums"."updateId") "albums"."id",
"assets"."ownerId", "albums"."ownerId",
"assets"."originalFileName", "albums"."albumName" as "name",
"assets"."thumbhash", "albums"."description",
"assets"."checksum", "albums"."createdAt",
"assets"."fileCreatedAt", "albums"."updatedAt",
"assets"."fileModifiedAt", "albums"."albumThumbnailAssetId" as "thumbnailAssetId",
"assets"."localDateTime", "albums"."isActivityEnabled",
"assets"."type", "albums"."order",
"assets"."deletedAt", "albums"."updateId"
"assets"."isFavorite",
"assets"."visibility",
"assets"."duration",
"assets"."updateId"
from
"assets"
where
"ownerId" = $1
and "updatedAt" < now() - interval '1 millisecond'
order by
"updateId" asc
-- SyncRepository.getPartnerBackfill
select
"sharedById",
"createId"
from from
"partners" "albums"
left join "albums_shared_users_users" as "album_users" on "albums"."id" = "album_users"."albumsId"
where where
"sharedWithId" = $1 "albums"."updatedAt" < now() - interval '1 millisecond'
and "createId" >= $2 and (
and "createdAt" < now() - interval '1 millisecond' "albums"."ownerId" = $1
or "album_users"."usersId" = $2
)
order by order by
"partners"."createId" asc "albums"."updateId" asc
-- SyncRepository.getPartnerAssetsBackfill -- SyncRepository.albumAsset.getBackfill
select select
"assets"."id", "assets"."id",
"assets"."ownerId", "assets"."ownerId",
@ -127,15 +67,16 @@ select
"assets"."updateId" "assets"."updateId"
from from
"assets" "assets"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "assets"."id"
where where
"ownerId" = $1 "album_assets"."albumsId" = $1
and "updatedAt" < now() - interval '1 millisecond' and "assets"."updatedAt" < now() - interval '1 millisecond'
and "updateId" <= $2 and "assets"."updateId" <= $2
and "updateId" >= $3 and "assets"."updateId" >= $3
order by order by
"updateId" asc "assets"."updateId" asc
-- SyncRepository.getPartnerAssetsUpserts -- SyncRepository.albumAsset.getUpserts
select select
"assets"."id", "assets"."id",
"assets"."ownerId", "assets"."ownerId",
@ -153,51 +94,19 @@ select
"assets"."updateId" "assets"."updateId"
from from
"assets" "assets"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "assets"."id"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where where
"ownerId" in ( "assets"."updatedAt" < now() - interval '1 millisecond'
select and (
"sharedById" "albums"."ownerId" = $1
from or "album_users"."usersId" = $2
"partners"
where
"sharedWithId" = $1
) )
and "updatedAt" < now() - interval '1 millisecond'
order by
"updateId" asc
-- SyncRepository.getAssetDeletes
select
"id",
"assetId"
from
"assets_audit"
where
"ownerId" = $1
and "deletedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "assets"."updateId" asc
-- SyncRepository.getPartnerAssetDeletes
select
"id",
"assetId"
from
"assets_audit"
where
"ownerId" in (
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.getAssetExifsUpserts -- SyncRepository.albumAssetExif.getBackfill
select select
"exif"."assetId", "exif"."assetId",
"exif"."description", "exif"."description",
@ -227,20 +136,16 @@ select
"exif"."updateId" "exif"."updateId"
from from
"exif" "exif"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "exif"."assetId"
where where
"assetId" in ( "album_assets"."albumsId" = $1
select and "exif"."updatedAt" < now() - interval '1 millisecond'
"id" and "exif"."updateId" <= $2
from and "exif"."updateId" >= $3
"assets"
where
"ownerId" = $1
)
and "updatedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "exif"."updateId" asc
-- SyncRepository.getPartnerAssetExifsBackfill -- SyncRepository.albumAssetExif.getUpserts
select select
"exif"."assetId", "exif"."assetId",
"exif"."description", "exif"."description",
@ -270,16 +175,192 @@ select
"exif"."updateId" "exif"."updateId"
from from
"exif" "exif"
inner join "assets" on "assets"."id" = "exif"."assetId" inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "exif"."assetId"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where where
"assets"."ownerId" = $1 "exif"."updatedAt" < now() - interval '1 millisecond'
and "exif"."updatedAt" < now() - interval '1 millisecond' and (
and "exif"."updateId" <= $2 "albums"."ownerId" = $1
and "exif"."updateId" >= $3 or "album_users"."usersId" = $2
)
order by order by
"exif"."updateId" asc "exif"."updateId" asc
-- SyncRepository.getPartnerAssetExifsUpserts -- SyncRepository.albumToAsset.getBackfill
select
"album_assets"."assetsId" as "assetId",
"album_assets"."albumsId" as "albumId",
"album_assets"."updateId"
from
"albums_assets_assets" as "album_assets"
where
"album_assets"."albumsId" = $1
and "album_assets"."updatedAt" < now() - interval '1 millisecond'
and "album_assets"."updateId" <= $2
and "album_assets"."updateId" >= $3
order by
"album_assets"."updateId" asc
-- SyncRepository.albumToAsset.getDeletes
select
"id",
"assetId",
"albumId"
from
"album_assets_audit"
where
"albumId" in (
select
"id"
from
"albums"
where
"ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.albumToAsset.getUpserts
select
"album_assets"."assetsId" as "assetId",
"album_assets"."albumsId" as "albumId",
"album_assets"."updateId"
from
"albums_assets_assets" as "album_assets"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where
"album_assets"."updatedAt" < now() - interval '1 millisecond'
and (
"albums"."ownerId" = $1
or "album_users"."usersId" = $2
)
order by
"album_assets"."updateId" asc
-- SyncRepository.albumUser.getBackfill
select
"album_users"."albumsId" as "albumId",
"album_users"."usersId" as "userId",
"album_users"."role",
"album_users"."updateId"
from
"albums_shared_users_users" as "album_users"
where
"albumsId" = $1
and "updatedAt" < now() - interval '1 millisecond'
and "updateId" <= $2
and "updateId" >= $3
order by
"updateId" asc
-- SyncRepository.albumUser.getDeletes
select
"id",
"userId",
"albumId"
from
"album_users_audit"
where
"albumId" in (
select
"id"
from
"albums"
where
"ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.albumUser.getUpserts
select
"album_users"."albumsId" as "albumId",
"album_users"."usersId" as "userId",
"album_users"."role",
"album_users"."updateId"
from
"albums_shared_users_users" as "album_users"
where
"album_users"."updatedAt" < now() - interval '1 millisecond'
and "album_users"."albumsId" in (
select
"id"
from
"albums"
where
"ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
)
order by
"album_users"."updateId" asc
-- SyncRepository.asset.getDeletes
select
"id",
"assetId"
from
"assets_audit"
where
"ownerId" = $1
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.asset.getUpserts
select
"assets"."id",
"assets"."ownerId",
"assets"."originalFileName",
"assets"."thumbhash",
"assets"."checksum",
"assets"."fileCreatedAt",
"assets"."fileModifiedAt",
"assets"."localDateTime",
"assets"."type",
"assets"."deletedAt",
"assets"."isFavorite",
"assets"."visibility",
"assets"."duration",
"assets"."updateId"
from
"assets"
where
"ownerId" = $1
and "updatedAt" < now() - interval '1 millisecond'
order by
"updateId" asc
-- SyncRepository.assetExif.getUpserts
select select
"exif"."assetId", "exif"."assetId",
"exif"."description", "exif"."description",
@ -316,173 +397,134 @@ where
from from
"assets" "assets"
where where
"ownerId" in ( "ownerId" = $1
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
) )
and "updatedAt" < now() - interval '1 millisecond' and "updatedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "updateId" asc
-- SyncRepository.getAlbumDeletes -- SyncRepository.memory.getDeletes
select select
"id", "id",
"albumId" "memoryId"
from from
"albums_audit" "memories_audit"
where where
"userId" = $1 "userId" = $1
and "deletedAt" < now() - interval '1 millisecond' and "deletedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "id" asc
-- SyncRepository.getAlbumUpserts -- SyncRepository.memory.getUpserts
select distinct select
on ("albums"."id", "albums"."updateId") "albums"."id", "id",
"albums"."ownerId", "createdAt",
"albums"."albumName" as "name", "updatedAt",
"albums"."description", "deletedAt",
"albums"."createdAt", "ownerId",
"albums"."updatedAt", "type",
"albums"."albumThumbnailAssetId" as "thumbnailAssetId", "data",
"albums"."isActivityEnabled", "isSaved",
"albums"."order", "memoryAt",
"albums"."updateId" "seenAt",
"showAt",
"hideAt",
"updateId"
from from
"albums" "memories"
left join "albums_shared_users_users" as "album_users" on "albums"."id" = "album_users"."albumsId"
where where
"albums"."updatedAt" < now() - interval '1 millisecond' "ownerId" = $1
and ( and "updatedAt" < now() - interval '1 millisecond'
"albums"."ownerId" = $1
or "album_users"."usersId" = $2
)
order by order by
"albums"."updateId" asc "updateId" asc
-- SyncRepository.getAlbumToAssetDeletes -- SyncRepository.memoryToAsset.getDeletes
select select
"id", "id",
"assetId", "memoryId",
"albumId" "assetId"
from from
"album_assets_audit" "memory_assets_audit"
where where
"albumId" in ( "memoryId" in (
select select
"id" "id"
from from
"albums" "memories"
where where
"ownerId" = $1 "ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
) )
and "deletedAt" < now() - interval '1 millisecond' and "deletedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "id" asc
-- SyncRepository.getAlbumUserDeletes -- SyncRepository.memoryToAsset.getUpserts
select select
"id", "memoriesId" as "memoryId",
"userId", "assetsId" as "assetId",
"albumId" "updateId"
from from
"album_users_audit" "memories_assets_assets"
where where
"albumId" in ( "memoriesId" in (
select select
"id" "id"
from from
"albums" "memories"
where where
"ownerId" = $1 "ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
) )
and "deletedAt" < now() - interval '1 millisecond' and "updatedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "updateId" asc
-- SyncRepository.getAlbumBackfill -- SyncRepository.partner.getCreatedAfter
select select
"albumsId" as "id", "sharedById",
"createId" "createId"
from from
"albums_shared_users_users" "partners"
where where
"usersId" = $1 "sharedWithId" = $1
and "createId" >= $2 and "createId" >= $2
and "createdAt" < now() - interval '1 millisecond' and "createdAt" < now() - interval '1 millisecond'
order by order by
"createId" asc "partners"."createId" asc
-- SyncRepository.getAlbumUsersBackfill -- SyncRepository.partner.getDeletes
select select
"album_users"."albumsId" as "albumId", "id",
"album_users"."usersId" as "userId", "sharedById",
"album_users"."role", "sharedWithId"
"album_users"."updateId"
from from
"albums_shared_users_users" as "album_users" "partners_audit"
where where
"albumsId" = $1 (
and "updatedAt" < now() - interval '1 millisecond' "sharedById" = $1
and "updateId" <= $2 or "sharedWithId" = $2
and "updateId" >= $3 )
and "deletedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "id" asc
-- SyncRepository.getAlbumUserUpserts -- SyncRepository.partner.getUpserts
select select
"album_users"."albumsId" as "albumId", "sharedById",
"album_users"."usersId" as "userId", "sharedWithId",
"album_users"."role", "inTimeline",
"album_users"."updateId" "updateId"
from from
"albums_shared_users_users" as "album_users" "partners"
where where
"album_users"."updatedAt" < now() - interval '1 millisecond' (
and "album_users"."albumsId" in ( "sharedById" = $1
select or "sharedWithId" = $2
"id"
from
"albums"
where
"ownerId" = $1
union
(
select
"albumUsers"."albumsId" as "id"
from
"albums_shared_users_users" as "albumUsers"
where
"albumUsers"."usersId" = $2
)
) )
and "updatedAt" < now() - interval '1 millisecond'
order by order by
"album_users"."updateId" asc "updateId" asc
-- SyncRepository.getAlbumAssetsBackfill -- SyncRepository.partnerAsset.getBackfill
select select
"assets"."id", "assets"."id",
"assets"."ownerId", "assets"."ownerId",
@ -500,16 +542,34 @@ select
"assets"."updateId" "assets"."updateId"
from from
"assets" "assets"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "assets"."id"
where where
"album_assets"."albumsId" = $1 "ownerId" = $1
and "assets"."updatedAt" < now() - interval '1 millisecond' and "updatedAt" < now() - interval '1 millisecond'
and "assets"."updateId" <= $2 and "updateId" <= $2
and "assets"."updateId" >= $3 and "updateId" >= $3
order by order by
"assets"."updateId" asc "updateId" asc
-- SyncRepository.partnerAsset.getDeletes
select
"id",
"assetId"
from
"assets_audit"
where
"ownerId" in (
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.getAlbumAssetsUpserts -- SyncRepository.partnerAsset.getUpserts
select select
"assets"."id", "assets"."id",
"assets"."ownerId", "assets"."ownerId",
@ -527,52 +587,20 @@ select
"assets"."updateId" "assets"."updateId"
from from
"assets" "assets"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "assets"."id"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where
"assets"."updatedAt" < now() - interval '1 millisecond'
and (
"albums"."ownerId" = $1
or "album_users"."usersId" = $2
)
order by
"assets"."updateId" asc
-- SyncRepository.getAlbumToAssetBackfill
select
"album_assets"."assetsId" as "assetId",
"album_assets"."albumsId" as "albumId",
"album_assets"."updateId"
from
"albums_assets_assets" as "album_assets"
where
"album_assets"."albumsId" = $1
and "album_assets"."updatedAt" < now() - interval '1 millisecond'
and "album_assets"."updateId" <= $2
and "album_assets"."updateId" >= $3
order by
"album_assets"."updateId" asc
-- SyncRepository.getAlbumToAssetUpserts
select
"album_assets"."assetsId" as "assetId",
"album_assets"."albumsId" as "albumId",
"album_assets"."updateId"
from
"albums_assets_assets" as "album_assets"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where where
"album_assets"."updatedAt" < now() - interval '1 millisecond' "ownerId" in (
and ( select
"albums"."ownerId" = $1 "sharedById"
or "album_users"."usersId" = $2 from
"partners"
where
"sharedWithId" = $1
) )
and "updatedAt" < now() - interval '1 millisecond'
order by order by
"album_assets"."updateId" asc "updateId" asc
-- SyncRepository.getAlbumAssetExifsBackfill -- SyncRepository.partnerAssetExif.getBackfill
select select
"exif"."assetId", "exif"."assetId",
"exif"."description", "exif"."description",
@ -602,16 +630,16 @@ select
"exif"."updateId" "exif"."updateId"
from from
"exif" "exif"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "exif"."assetId" inner join "assets" on "assets"."id" = "exif"."assetId"
where where
"album_assets"."albumsId" = $1 "assets"."ownerId" = $1
and "exif"."updatedAt" < now() - interval '1 millisecond' and "exif"."updatedAt" < now() - interval '1 millisecond'
and "exif"."updateId" <= $2 and "exif"."updateId" <= $2
and "exif"."updateId" >= $3 and "exif"."updateId" >= $3
order by order by
"exif"."updateId" asc "exif"."updateId" asc
-- SyncRepository.getAlbumAssetExifsUpserts -- SyncRepository.partnerAssetExif.getUpserts
select select
"exif"."assetId", "exif"."assetId",
"exif"."description", "exif"."description",
@ -641,89 +669,47 @@ select
"exif"."updateId" "exif"."updateId"
from from
"exif" "exif"
inner join "albums_assets_assets" as "album_assets" on "album_assets"."assetsId" = "exif"."assetId"
inner join "albums" on "albums"."id" = "album_assets"."albumsId"
left join "albums_shared_users_users" as "album_users" on "album_users"."albumsId" = "album_assets"."albumsId"
where where
"exif"."updatedAt" < now() - interval '1 millisecond' "assetId" in (
and ( select
"albums"."ownerId" = $1 "id"
or "album_users"."usersId" = $2 from
"assets"
where
"ownerId" in (
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
) )
order by
"exif"."updateId" asc
-- SyncRepository.getMemoryUpserts
select
"id",
"createdAt",
"updatedAt",
"deletedAt",
"ownerId",
"type",
"data",
"isSaved",
"memoryAt",
"seenAt",
"showAt",
"hideAt",
"updateId"
from
"memories"
where
"ownerId" = $1
and "updatedAt" < now() - interval '1 millisecond' and "updatedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "updateId" asc
-- SyncRepository.getMemoryDeletes -- SyncRepository.user.getDeletes
select select
"id", "id",
"memoryId" "userId"
from from
"memories_audit" "users_audit"
where where
"userId" = $1 "deletedAt" < now() - interval '1 millisecond'
and "deletedAt" < now() - interval '1 millisecond'
order by order by
"id" asc "id" asc
-- SyncRepository.getMemoryAssetUpserts -- SyncRepository.user.getUpserts
select select
"memoriesId" as "memoryId", "id",
"assetsId" as "assetId", "name",
"email",
"deletedAt",
"updateId" "updateId"
from from
"memories_assets_assets" "users"
where where
"memoriesId" in ( "updatedAt" < now() - interval '1 millisecond'
select
"id"
from
"memories"
where
"ownerId" = $1
)
and "updatedAt" < now() - interval '1 millisecond'
order by order by
"updateId" asc "updateId" asc
-- SyncRepository.getMemoryAssetDeletes
select
"id",
"memoryId",
"assetId"
from
"memory_assets_audit"
where
"memoryId" in (
select
"id"
from
"memories"
where
"ownerId" = $1
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc

2
server/src/repositories/index.ts
Unescape Escape View File

@ -34,6 +34,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -81,6 +82,7 @@ export const repositories = [
StackRepository, StackRepository,
StorageRepository, StorageRepository,
SyncRepository, SyncRepository,
SyncCheckpointRepository,
SystemMetadataRepository, SystemMetadataRepository,
TagRepository, TagRepository,
TelemetryRepository, TelemetryRepository,

41
server/src/repositories/sync-checkpoint.repository.ts
Unescape Escape View File

@ -0,0 +1,41 @@
import { Injectable } from '@nestjs/common';
import { Insertable, Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB, SessionSyncCheckpoints } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { SyncEntityType } from 'src/enum';
@Injectable()
export class SyncCheckpointRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
getAll(sessionId: string) {
return this.db
.selectFrom('session_sync_checkpoints')
.select(['type', 'ack'])
.where('sessionId', '=', sessionId)
.execute();
}
upsertAll(items: Insertable<SessionSyncCheckpoints>[]) {
return this.db
.insertInto('session_sync_checkpoints')
.values(items)
.onConflict((oc) =>
oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({
ack: eb.ref('excluded.ack'),
})),
)
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
deleteAll(sessionId: string, types?: SyncEntityType[]) {
return this.db
.deleteFrom('session_sync_checkpoints')
.where('sessionId', '=', sessionId)
.$if(!!types, (qb) => qb.where('type', 'in', types!))
.execute();
}
}

590
server/src/repositories/sync.repository.ts
Unescape Escape View File

@ -1,10 +1,9 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { Insertable, Kysely, SelectQueryBuilder, sql } from 'kysely'; import { Kysely, SelectQueryBuilder, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely'; import { InjectKysely } from 'nestjs-kysely';
import { columns } from 'src/database'; import { columns } from 'src/database';
import { DB, SessionSyncCheckpoints } from 'src/db'; import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators'; import { DummyValue, GenerateSql } from 'src/decorators';
import { SyncEntityType } from 'src/enum';
import { SyncAck } from 'src/types'; import { SyncAck } from 'src/types';
type AuditTables = type AuditTables =
@ -28,167 +27,154 @@ type UpsertTables =
@Injectable() @Injectable()
export class SyncRepository { export class SyncRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {} album: AlbumSync;
albumAsset: AlbumAssetSync;
@GenerateSql({ params: [DummyValue.UUID] }) albumAssetExif: AlbumAssetExifSync;
getCheckpoints(sessionId: string) { albumToAsset: AlbumToAssetSync;
return this.db albumUser: AlbumUserSync;
.selectFrom('session_sync_checkpoints') asset: AssetSync;
.select(['type', 'ack']) assetExif: AssetExifSync;
.where('sessionId', '=', sessionId) memory: MemorySync;
.execute(); memoryToAsset: MemoryToAssetSync;
} partner: PartnerSync;
partnerAsset: PartnerAssetsSync;
upsertCheckpoints(items: Insertable<SessionSyncCheckpoints>[]) { partnerAssetExif: PartnerAssetExifsSync;
return this.db user: UserSync;
.insertInto('session_sync_checkpoints')
.values(items) constructor(@InjectKysely() private db: Kysely<DB>) {
.onConflict((oc) => this.album = new AlbumSync(this.db);
oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({ this.albumAsset = new AlbumAssetSync(this.db);
ack: eb.ref('excluded.ack'), this.albumAssetExif = new AlbumAssetExifSync(this.db);
})), this.albumToAsset = new AlbumToAssetSync(this.db);
) this.albumUser = new AlbumUserSync(this.db);
.execute(); this.asset = new AssetSync(this.db);
this.assetExif = new AssetExifSync(this.db);
this.memory = new MemorySync(this.db);
this.memoryToAsset = new MemoryToAssetSync(this.db);
this.partner = new PartnerSync(this.db);
this.partnerAsset = new PartnerAssetsSync(this.db);
this.partnerAssetExif = new PartnerAssetExifsSync(this.db);
this.user = new UserSync(this.db);
} }
}
@GenerateSql({ params: [DummyValue.UUID] }) class BaseSync {
deleteCheckpoints(sessionId: string, types?: SyncEntityType[]) { constructor(protected db: Kysely<DB>) {}
return this.db
.deleteFrom('session_sync_checkpoints')
.where('sessionId', '=', sessionId)
.$if(!!types, (qb) => qb.where('type', 'in', types!))
.execute();
}
@GenerateSql({ params: [], stream: true }) protected auditTableFilters<T extends keyof Pick<DB, AuditTables>, D>(
getUserUpserts(ack?: SyncAck) { qb: SelectQueryBuilder<DB, T, D>,
return this.db ack?: SyncAck,
.selectFrom('users') ) {
.select(['id', 'name', 'email', 'deletedAt', 'updateId']) const builder = qb as SelectQueryBuilder<DB, AuditTables, D>;
.$call((qb) => this.upsertTableFilters(qb, ack)) return builder
.stream(); .where('deletedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId))
.orderBy('id', 'asc') as SelectQueryBuilder<DB, T, D>;
} }
@GenerateSql({ params: [], stream: true }) protected upsertTableFilters<T extends keyof Pick<DB, UpsertTables>, D>(
getUserDeletes(ack?: SyncAck) { qb: SelectQueryBuilder<DB, T, D>,
return this.db ack?: SyncAck,
.selectFrom('users_audit') ) {
.select(['id', 'userId']) const builder = qb as SelectQueryBuilder<DB, UpsertTables, D>;
.$call((qb) => this.auditTableFilters(qb, ack)) return builder
.stream(); .where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId))
.orderBy('updateId', 'asc') as SelectQueryBuilder<DB, T, D>;
} }
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) class AlbumSync extends BaseSync {
getPartnerUpserts(userId: string, ack?: SyncAck) { @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
getCreatedAfter(userId: string, afterCreateId?: string) {
return this.db return this.db
.selectFrom('partners') .selectFrom('albums_shared_users_users')
.select(['sharedById', 'sharedWithId', 'inTimeline', 'updateId']) .select(['albumsId as id', 'createId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)])) .where('usersId', '=', userId)
.$call((qb) => this.upsertTableFilters(qb, ack)) .$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.stream(); .where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('createId', 'asc')
.execute();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerDeletes(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('partners_audit') .selectFrom('albums_audit')
.select(['id', 'sharedById', 'sharedWithId']) .select(['id', 'albumId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)])) .where('userId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack)) .$call((qb) => this.auditTableFilters(qb, ack))
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('assets') .selectFrom('albums')
.select(columns.syncAsset) .distinctOn(['albums.id', 'albums.updateId'])
.select('assets.updateId') .where('albums.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('ownerId', '=', userId) .$if(!!ack, (qb) => qb.where('albums.updateId', '>', ack!.updateId))
.$call((qb) => this.upsertTableFilters(qb, ack)) .orderBy('albums.updateId', 'asc')
.leftJoin('albums_shared_users_users as album_users', 'albums.id', 'album_users.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.select([
'albums.id',
'albums.ownerId',
'albums.albumName as name',
'albums.description',
'albums.createdAt',
'albums.updatedAt',
'albums.albumThumbnailAssetId as thumbnailAssetId',
'albums.isActivityEnabled',
'albums.order',
'albums.updateId',
])
.stream(); .stream();
} }
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) class AlbumAssetSync extends BaseSync {
getPartnerBackfill(userId: string, afterCreateId?: string) {
return this.db
.selectFrom('partners')
.select(['sharedById', 'createId'])
.where('sharedWithId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('partners.createId', 'asc')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getPartnerAssetsBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) { getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db return this.db
.selectFrom('assets') .selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset) .select(columns.syncAsset)
.select('assets.updateId') .select('assets.updateId')
.where('ownerId', '=', partnerId) .where('album_assets.albumsId', '=', albumId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId) .where('assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!)) .$if(!!afterUpdateId, (eb) => eb.where('assets.updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc') .orderBy('assets.updateId', 'asc')
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetsUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('assets') .selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset) .select(columns.syncAsset)
.select('assets.updateId') .select('assets.updateId')
.where('ownerId', 'in', (eb) => .where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId), .$if(!!ack, (qb) => qb.where('assets.updateId', '>', ack!.updateId))
) .orderBy('assets.updateId', 'asc')
.$call((qb) => this.upsertTableFilters(qb, ack)) .innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.stream(); .leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
} .where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetExifsUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('assetId', 'in', (eb) => eb.selectFrom('assets').select('id').where('ownerId', '=', userId))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream(); .stream();
} }
}
class AlbumAssetExifSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getPartnerAssetExifsBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) { getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db return this.db
.selectFrom('exif') .selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif) .select(columns.syncAssetExif)
.select('exif.updateId') .select('exif.updateId')
.innerJoin('assets', 'assets.id', 'exif.assetId') .where('album_assets.albumsId', '=', albumId)
.where('assets.ownerId', '=', partnerId)
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('exif.updateId', '<=', beforeUpdateId) .where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!)) .$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
@ -197,60 +183,38 @@ export class SyncRepository {
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetExifsUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('exif') .selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif) .select(columns.syncAssetExif)
.select('exif.updateId') .select('exif.updateId')
.where('assetId', 'in', (eb) => .where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
eb .$if(!!ack, (qb) => qb.where('exif.updateId', '>', ack!.updateId))
.selectFrom('assets') .orderBy('exif.updateId', 'asc')
.select('id') .innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.where('ownerId', 'in', (eb) => .leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId), .where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_audit')
.select(['id', 'albumId'])
.where('userId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream(); .stream();
} }
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) class AlbumToAssetSync extends BaseSync {
getAlbumUpserts(userId: string, ack?: SyncAck) { @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db return this.db
.selectFrom('albums') .selectFrom('albums_assets_assets as album_assets')
.distinctOn(['albums.id', 'albums.updateId']) .select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('albums.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('album_assets.albumsId', '=', albumId)
.$if(!!ack, (qb) => qb.where('albums.updateId', '>', ack!.updateId)) .where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('albums.updateId', 'asc') .where('album_assets.updateId', '<=', beforeUpdateId)
.leftJoin('albums_shared_users_users as album_users', 'albums.id', 'album_users.albumsId') .$if(!!afterUpdateId, (eb) => eb.where('album_assets.updateId', '>=', afterUpdateId!))
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)])) .orderBy('album_assets.updateId', 'asc')
.select([
'albums.id',
'albums.ownerId',
'albums.albumName as name',
'albums.description',
'albums.createdAt',
'albums.updatedAt',
'albums.albumThumbnailAssetId as thumbnailAssetId',
'albums.isActivityEnabled',
'albums.order',
'albums.updateId',
])
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumToAssetDeletes(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('album_assets_audit') .selectFrom('album_assets_audit')
.select(['id', 'assetId', 'albumId']) .select(['id', 'assetId', 'albumId'])
@ -277,7 +241,37 @@ export class SyncRepository {
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUserDeletes(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('album_assets.updateId', '>', ack!.updateId))
.orderBy('album_assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
}
class AlbumUserSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser)
.select('album_users.updateId')
.where('albumsId', '=', albumId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('album_users_audit') .selectFrom('album_users_audit')
.select(['id', 'userId', 'albumId']) .select(['id', 'userId', 'albumId'])
@ -303,34 +297,8 @@ export class SyncRepository {
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
getAlbumBackfill(userId: string, afterCreateId?: string) {
return this.db
.selectFrom('albums_shared_users_users')
.select(['albumsId as id', 'createId'])
.where('usersId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('createId', 'asc')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumUsersBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser)
.select('album_users.updateId')
.where('albumsId', '=', albumId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUserUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('albums_shared_users_users as album_users') .selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser) .select(columns.syncAlbumUser)
@ -358,98 +326,57 @@ export class SyncRepository {
) )
.stream(); .stream();
} }
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true }) class AssetSync extends BaseSync {
getAlbumAssetsBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) { @GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('assets') .selectFrom('assets_audit')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id') .select(['id', 'assetId'])
.select(columns.syncAsset) .where('ownerId', '=', userId)
.select('assets.updateId') .$call((qb) => this.auditTableFilters(qb, ack))
.where('album_assets.albumsId', '=', albumId)
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('assets.updateId', '>=', afterUpdateId!))
.orderBy('assets.updateId', 'asc')
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumAssetsUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('assets') .selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset) .select(columns.syncAsset)
.select('assets.updateId') .select('assets.updateId')
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('ownerId', '=', userId)
.$if(!!ack, (qb) => qb.where('assets.updateId', '>', ack!.updateId)) .$call((qb) => this.upsertTableFilters(qb, ack))
.orderBy('assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumToAssetBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.albumsId', '=', albumId)
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('album_assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('album_assets.updateId', '>=', afterUpdateId!))
.orderBy('album_assets.updateId', 'asc')
.stream(); .stream();
} }
}
class AssetExifSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumToAssetUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('album_assets.updateId', '>', ack!.updateId))
.orderBy('album_assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumAssetExifsBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db return this.db
.selectFrom('exif') .selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif) .select(columns.syncAssetExif)
.select('exif.updateId') .select('exif.updateId')
.where('album_assets.albumsId', '=', albumId) .where('assetId', 'in', (eb) => eb.selectFrom('assets').select('id').where('ownerId', '=', userId))
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .$call((qb) => this.upsertTableFilters(qb, ack))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream(); .stream();
} }
}
class MemorySync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumAssetExifsUpserts(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('exif') .selectFrom('memories_audit')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId') .select(['id', 'memoryId'])
.select(columns.syncAssetExif) .where('userId', '=', userId)
.select('exif.updateId') .$call((qb) => this.auditTableFilters(qb, ack))
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('exif.updateId', '>', ack!.updateId))
.orderBy('exif.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memories') .selectFrom('memories')
.select([ .select([
@ -471,19 +398,21 @@ export class SyncRepository {
.$call((qb) => this.upsertTableFilters(qb, ack)) .$call((qb) => this.upsertTableFilters(qb, ack))
.stream(); .stream();
} }
}
class MemoryToAssetSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryDeletes(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memories_audit') .selectFrom('memory_assets_audit')
.select(['id', 'memoryId']) .select(['id', 'memoryId', 'assetId'])
.where('userId', '=', userId) .where('memoryId', 'in', (eb) => eb.selectFrom('memories').select('id').where('ownerId', '=', userId))
.$call((qb) => this.auditTableFilters(qb, ack)) .$call((qb) => this.auditTableFilters(qb, ack))
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryAssetUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memories_assets_assets') .selectFrom('memories_assets_assets')
.select(['memoriesId as memoryId', 'assetsId as assetId']) .select(['memoriesId as memoryId', 'assetsId as assetId'])
@ -492,33 +421,134 @@ export class SyncRepository {
.$call((qb) => this.upsertTableFilters(qb, ack)) .$call((qb) => this.upsertTableFilters(qb, ack))
.stream(); .stream();
} }
}
class PartnerSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
getCreatedAfter(userId: string, afterCreateId?: string) {
return this.db
.selectFrom('partners')
.select(['sharedById', 'createId'])
.where('sharedWithId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('partners.createId', 'asc')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryAssetDeletes(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memory_assets_audit') .selectFrom('partners_audit')
.select(['id', 'memoryId', 'assetId']) .select(['id', 'sharedById', 'sharedWithId'])
.where('memoryId', 'in', (eb) => eb.selectFrom('memories').select('id').where('ownerId', '=', userId)) .where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.auditTableFilters(qb, ack)) .$call((qb) => this.auditTableFilters(qb, ack))
.stream(); .stream();
} }
private auditTableFilters<T extends keyof Pick<DB, AuditTables>, D>(qb: SelectQueryBuilder<DB, T, D>, ack?: SyncAck) { @GenerateSql({ params: [DummyValue.UUID], stream: true })
const builder = qb as SelectQueryBuilder<DB, AuditTables, D>; getUpserts(userId: string, ack?: SyncAck) {
return builder return this.db
.where('deletedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .selectFrom('partners')
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) .select(['sharedById', 'sharedWithId', 'inTimeline', 'updateId'])
.orderBy('id', 'asc') as SelectQueryBuilder<DB, T, D>; .where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
} }
}
private upsertTableFilters<T extends keyof Pick<DB, UpsertTables>, D>( class PartnerAssetsSync extends BaseSync {
qb: SelectQueryBuilder<DB, T, D>, @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
ack?: SyncAck, getBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
) { return this.db
const builder = qb as SelectQueryBuilder<DB, UpsertTables, D>; .selectFrom('assets')
return builder .select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', '=', partnerId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId)) .where('updateId', '<=', beforeUpdateId)
.orderBy('updateId', 'asc') as SelectQueryBuilder<DB, T, D>; .$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class PartnerAssetExifsSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', partnerId)
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('assetId', 'in', (eb) =>
eb
.selectFrom('assets')
.select('id')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class UserSync extends BaseSync {
@GenerateSql({ params: [], stream: true })
getDeletes(ack?: SyncAck) {
return this.db
.selectFrom('users_audit')
.select(['id', 'userId'])
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [], stream: true })
getUpserts(ack?: SyncAck) {
return this.db
.selectFrom('users')
.select(['id', 'name', 'email', 'deletedAt', 'updateId'])
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
} }
} }

3
server/src/services/base.service.ts
Unescape Escape View File

@ -41,6 +41,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -91,6 +92,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
StackRepository, StackRepository,
StorageRepository, StorageRepository,
SyncRepository, SyncRepository,
SyncCheckpointRepository,
SystemMetadataRepository, SystemMetadataRepository,
TagRepository, TagRepository,
TelemetryRepository, TelemetryRepository,
@ -142,6 +144,7 @@ export class BaseService {
protected stackRepository: StackRepository, protected stackRepository: StackRepository,
protected storageRepository: StorageRepository, protected storageRepository: StorageRepository,
protected syncRepository: SyncRepository, protected syncRepository: SyncRepository,
protected syncCheckpointRepository: SyncCheckpointRepository,
protected systemMetadataRepository: SystemMetadataRepository, protected systemMetadataRepository: SystemMetadataRepository,
protected tagRepository: TagRepository, protected tagRepository: TagRepository,
protected telemetryRepository: TelemetryRepository, protected telemetryRepository: TelemetryRepository,

78
server/src/services/sync.service.ts
Unescape Escape View File

@ -81,7 +81,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
return this.syncRepository.getCheckpoints(sessionId); return this.syncCheckpointRepository.getAll(sessionId);
} }
async setAcks(auth: AuthDto, dto: SyncAckSetDto) { async setAcks(auth: AuthDto, dto: SyncAckSetDto) {
@ -102,7 +102,7 @@ export class SyncService extends BaseService {
checkpoints[type] = { sessionId, type, ack }; checkpoints[type] = { sessionId, type, ack };
} }
await this.syncRepository.upsertCheckpoints(Object.values(checkpoints)); await this.syncCheckpointRepository.upsertAll(Object.values(checkpoints));
} }
async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) {
@ -111,7 +111,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
await this.syncRepository.deleteCheckpoints(sessionId, dto.types); await this.syncCheckpointRepository.deleteAll(sessionId, dto.types);
} }
async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) { async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) {
@ -120,7 +120,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
const checkpoints = await this.syncRepository.getCheckpoints(sessionId); const checkpoints = await this.syncCheckpointRepository.getAll(sessionId);
const checkpointMap: CheckpointMap = Object.fromEntries(checkpoints.map(({ type, ack }) => [type, fromAck(ack)])); const checkpointMap: CheckpointMap = Object.fromEntries(checkpoints.map(({ type, ack }) => [type, fromAck(ack)]));
const handlers: Record<SyncRequestType, () => Promise<void>> = { const handlers: Record<SyncRequestType, () => Promise<void>> = {
[SyncRequestType.UsersV1]: () => this.syncUsersV1(response, checkpointMap), [SyncRequestType.UsersV1]: () => this.syncUsersV1(response, checkpointMap),
@ -149,13 +149,13 @@ export class SyncService extends BaseService {
private async syncUsersV1(response: Writable, checkpointMap: CheckpointMap) { private async syncUsersV1(response: Writable, checkpointMap: CheckpointMap) {
const deleteType = SyncEntityType.UserDeleteV1; const deleteType = SyncEntityType.UserDeleteV1;
const deletes = this.syncRepository.getUserDeletes(checkpointMap[deleteType]); const deletes = this.syncRepository.user.getDeletes(checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.UserV1; const upsertType = SyncEntityType.UserV1;
const upserts = this.syncRepository.getUserUpserts(checkpointMap[upsertType]); const upserts = this.syncRepository.user.getUpserts(checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -163,13 +163,13 @@ export class SyncService extends BaseService {
private async syncPartnersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncPartnersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.PartnerDeleteV1; const deleteType = SyncEntityType.PartnerDeleteV1;
const deletes = this.syncRepository.getPartnerDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.partner.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.PartnerV1; const upsertType = SyncEntityType.PartnerV1;
const upserts = this.syncRepository.getPartnerUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partner.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -177,13 +177,13 @@ export class SyncService extends BaseService {
private async syncAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.AssetDeleteV1; const deleteType = SyncEntityType.AssetDeleteV1;
const deletes = this.syncRepository.getAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.asset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.AssetV1; const upsertType = SyncEntityType.AssetV1;
const upserts = this.syncRepository.getAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.asset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -196,14 +196,14 @@ export class SyncService extends BaseService {
sessionId: string, sessionId: string,
) { ) {
const deleteType = SyncEntityType.PartnerAssetDeleteV1; const deleteType = SyncEntityType.PartnerAssetDeleteV1;
const deletes = this.syncRepository.getPartnerAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.partnerAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.PartnerAssetBackfillV1; const backfillType = SyncEntityType.PartnerAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId); const partners = await this.syncRepository.partner.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.PartnerAssetV1; const upsertType = SyncEntityType.PartnerAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -216,7 +216,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getPartnerAssetsBackfill(partner.sharedById, startId, endId); const backfill = this.syncRepository.partnerAsset.getBackfill(partner.sharedById, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { send(response, {
@ -236,7 +236,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getPartnerAssetsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partnerAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -244,7 +244,7 @@ export class SyncService extends BaseService {
private async syncAssetExifsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAssetExifsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const upsertType = SyncEntityType.AssetExifV1; const upsertType = SyncEntityType.AssetExifV1;
const upserts = this.syncRepository.getAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.assetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -258,7 +258,7 @@ export class SyncService extends BaseService {
) { ) {
const backfillType = SyncEntityType.PartnerAssetExifBackfillV1; const backfillType = SyncEntityType.PartnerAssetExifBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId); const partners = await this.syncRepository.partner.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.PartnerAssetExifV1; const upsertType = SyncEntityType.PartnerAssetExifV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
@ -272,7 +272,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getPartnerAssetExifsBackfill(partner.sharedById, startId, endId); const backfill = this.syncRepository.partnerAssetExif.getBackfill(partner.sharedById, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [partner.createId, updateId], data }); send(response, { type: backfillType, ids: [partner.createId, updateId], data });
@ -288,7 +288,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getPartnerAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partnerAssetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -296,13 +296,13 @@ export class SyncService extends BaseService {
private async syncAlbumsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAlbumsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.AlbumDeleteV1; const deleteType = SyncEntityType.AlbumDeleteV1;
const deletes = this.syncRepository.getAlbumDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.album.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.AlbumV1; const upsertType = SyncEntityType.AlbumV1;
const upserts = this.syncRepository.getAlbumUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.album.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -310,14 +310,14 @@ export class SyncService extends BaseService {
private async syncAlbumUsersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) { private async syncAlbumUsersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) {
const deleteType = SyncEntityType.AlbumUserDeleteV1; const deleteType = SyncEntityType.AlbumUserDeleteV1;
const deletes = this.syncRepository.getAlbumUserDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.albumUser.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.AlbumUserBackfillV1; const backfillType = SyncEntityType.AlbumUserBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumUserV1; const upsertType = SyncEntityType.AlbumUserV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -330,7 +330,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumUsersBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumUser.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -346,7 +346,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumUserUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumUser.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -355,7 +355,7 @@ export class SyncService extends BaseService {
private async syncAlbumAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) { private async syncAlbumAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) {
const backfillType = SyncEntityType.AlbumAssetBackfillV1; const backfillType = SyncEntityType.AlbumAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumAssetV1; const upsertType = SyncEntityType.AlbumAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -368,7 +368,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumAssetsBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumAsset.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data: mapSyncAssetV1(data) }); send(response, { type: backfillType, ids: [createId, updateId], data: mapSyncAssetV1(data) });
@ -384,7 +384,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumAssetsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -398,7 +398,7 @@ export class SyncService extends BaseService {
) { ) {
const backfillType = SyncEntityType.AlbumAssetExifBackfillV1; const backfillType = SyncEntityType.AlbumAssetExifBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumAssetExifV1; const upsertType = SyncEntityType.AlbumAssetExifV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -411,7 +411,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumAssetExifsBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumAssetExif.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -427,7 +427,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumAssetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -440,14 +440,14 @@ export class SyncService extends BaseService {
sessionId: string, sessionId: string,
) { ) {
const deleteType = SyncEntityType.AlbumToAssetDeleteV1; const deleteType = SyncEntityType.AlbumToAssetDeleteV1;
const deletes = this.syncRepository.getAlbumToAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.albumToAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.AlbumToAssetBackfillV1; const backfillType = SyncEntityType.AlbumToAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumToAssetV1; const upsertType = SyncEntityType.AlbumToAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -460,7 +460,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumToAssetBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumToAsset.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -476,7 +476,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumToAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumToAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -484,13 +484,13 @@ export class SyncService extends BaseService {
private async syncMemoriesV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncMemoriesV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.MemoryDeleteV1; const deleteType = SyncEntityType.MemoryDeleteV1;
const deletes = this.syncRepository.getMemoryDeletes(auth.user.id, checkpointMap[SyncEntityType.MemoryDeleteV1]); const deletes = this.syncRepository.memory.getDeletes(auth.user.id, checkpointMap[SyncEntityType.MemoryDeleteV1]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.MemoryV1; const upsertType = SyncEntityType.MemoryV1;
const upserts = this.syncRepository.getMemoryUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.memory.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -498,13 +498,13 @@ export class SyncService extends BaseService {
private async syncMemoryAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncMemoryAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.MemoryToAssetDeleteV1; const deleteType = SyncEntityType.MemoryToAssetDeleteV1;
const deletes = this.syncRepository.getMemoryAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.memoryToAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.MemoryToAssetV1; const upsertType = SyncEntityType.MemoryToAssetV1;
const upserts = this.syncRepository.getMemoryAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.memoryToAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -512,7 +512,7 @@ export class SyncService extends BaseService {
private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) { private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) {
const { type, sessionId, createId } = item; const { type, sessionId, createId } = item;
await this.syncRepository.upsertCheckpoints([ await this.syncCheckpointRepository.upsertAll([
{ {
type, type,
sessionId, sessionId,

9
server/test/medium.factory.ts
Unescape Escape View File

@ -26,6 +26,7 @@ import { PersonRepository } from 'src/repositories/person.repository';
import { SearchRepository } from 'src/repositories/search.repository'; import { SearchRepository } from 'src/repositories/search.repository';
import { SessionRepository } from 'src/repositories/session.repository'; import { SessionRepository } from 'src/repositories/session.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { UserRepository } from 'src/repositories/user.repository'; import { UserRepository } from 'src/repositories/user.repository';
@ -202,7 +203,11 @@ export class MediumTestContext<S extends BaseService = BaseService> {
export class SyncTestContext extends MediumTestContext<SyncService> { export class SyncTestContext extends MediumTestContext<SyncService> {
constructor(database: Kysely<DB>) { constructor(database: Kysely<DB>) {
super(SyncService, { database, real: [SyncRepository, SessionRepository], mock: [LoggingRepository] }); super(SyncService, {
database,
real: [SyncRepository, SyncCheckpointRepository, SessionRepository],
mock: [LoggingRepository],
});
} }
async syncStream(auth: AuthDto, types: SyncRequestType[]) { async syncStream(auth: AuthDto, types: SyncRequestType[]) {
@ -239,6 +244,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
case SearchRepository: case SearchRepository:
case SessionRepository: case SessionRepository:
case SyncRepository: case SyncRepository:
case SyncCheckpointRepository:
case SystemMetadataRepository: case SystemMetadataRepository:
case UserRepository: case UserRepository:
case VersionHistoryRepository: { case VersionHistoryRepository: {
@ -282,6 +288,7 @@ const newMockRepository = <T>(key: ClassConstructor<T>) => {
case PersonRepository: case PersonRepository:
case SessionRepository: case SessionRepository:
case SyncRepository: case SyncRepository:
case SyncCheckpointRepository:
case SystemMetadataRepository: case SystemMetadataRepository:
case UserRepository: case UserRepository:
case VersionHistoryRepository: { case VersionHistoryRepository: {

4
server/test/utils.ts
Unescape Escape View File

@ -47,6 +47,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -217,6 +218,7 @@ export type ServiceOverrides = {
stack: StackRepository; stack: StackRepository;
storage: StorageRepository; storage: StorageRepository;
sync: SyncRepository; sync: SyncRepository;
syncCheckpoint: SyncCheckpointRepository;
systemMetadata: SystemMetadataRepository; systemMetadata: SystemMetadataRepository;
tag: TagRepository; tag: TagRepository;
telemetry: TelemetryRepository; telemetry: TelemetryRepository;
@ -287,6 +289,7 @@ export const newTestService = <T extends BaseService>(
stack: automock(StackRepository), stack: automock(StackRepository),
storage: newStorageRepositoryMock(), storage: newStorageRepositoryMock(),
sync: automock(SyncRepository), sync: automock(SyncRepository),
syncCheckpoint: automock(SyncCheckpointRepository),
systemMetadata: newSystemMetadataRepositoryMock(), systemMetadata: newSystemMetadataRepositoryMock(),
// systemMetadata: automock(SystemMetadataRepository, { strict: false }), // systemMetadata: automock(SystemMetadataRepository, { strict: false }),
// eslint-disable-next-line no-sparse-arrays // eslint-disable-next-line no-sparse-arrays
@ -336,6 +339,7 @@ export const newTestService = <T extends BaseService>(
overrides.stack || (mocks.stack as As<StackRepository>), overrides.stack || (mocks.stack as As<StackRepository>),
overrides.storage || (mocks.storage as As<StorageRepository>), overrides.storage || (mocks.storage as As<StorageRepository>),
overrides.sync || (mocks.sync as As<SyncRepository>), overrides.sync || (mocks.sync as As<SyncRepository>),
overrides.syncCheckpoint || (mocks.syncCheckpoint as As<SyncCheckpointRepository>),
overrides.systemMetadata || (mocks.systemMetadata as As<SystemMetadataRepository>), overrides.systemMetadata || (mocks.systemMetadata as As<SystemMetadataRepository>),
overrides.tag || (mocks.tag as As<TagRepository>), overrides.tag || (mocks.tag as As<TagRepository>),
overrides.telemetry || (mocks.telemetry as unknown as TelemetryRepository), overrides.telemetry || (mocks.telemetry as unknown as TelemetryRepository),