mirror of https://github.com/immich-app/immich.git
feat(server): efficient full app sync (#8755)
* feat(server): efficient full app sync * add SQL, fix test compile issues * fix linter warning * new sync controller+service, add tests * enable new sync controller+service * Update server/src/services/sync.service.ts Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>pull/8841/head
Fynn Petersen-Frey
GitHub
parent
58e516c766
commit
103cb60a57
@ -0,0 +1,17 @@
|
||||
# openapi.model.AssetDeltaSyncResponseDto
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**deleted** | **List<String>** | | [default to const []]
|
||||
**needsFullSync** | **bool** | |
|
||||
**upserted** | [**List<AssetResponseDto>**](AssetResponseDto.md) | | [default to const []]
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
@ -0,0 +1,135 @@
|
||||
# openapi.api.SyncApi
|
||||
|
||||
## Load the API package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
All URIs are relative to */api*
|
||||
|
||||
Method | HTTP request | Description
|
||||
------------- | ------------- | -------------
|
||||
[**getAllForUserFullSync**](SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync |
|
||||
[**getDeltaSync**](SyncApi.md#getdeltasync) | **GET** /sync/delta-sync |
|
||||
|
||||
|
||||
# **getAllForUserFullSync**
|
||||
> List<AssetResponseDto> getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId)
|
||||
|
||||
|
||||
|
||||
### Example
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
// TODO Configure API key authorization: cookie
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
|
||||
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
|
||||
// TODO Configure API key authorization: api_key
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
|
||||
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
|
||||
// TODO Configure HTTP Bearer authorization: bearer
|
||||
// Case 1. Use String Token
|
||||
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
|
||||
// Case 2. Use Function which generate token.
|
||||
// String yourTokenGeneratorFunction() { ... }
|
||||
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
|
||||
|
||||
final api_instance = SyncApi();
|
||||
final limit = 56; // int |
|
||||
final updatedUntil = 2013-10-20T19:20:30+01:00; // DateTime |
|
||||
final lastCreationDate = 2013-10-20T19:20:30+01:00; // DateTime |
|
||||
final lastId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String |
|
||||
final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String |
|
||||
|
||||
try {
|
||||
final result = api_instance.getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId);
|
||||
print(result);
|
||||
} catch (e) {
|
||||
print('Exception when calling SyncApi->getAllForUserFullSync: $e\n');
|
||||
}
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
Name | Type | Description | Notes
|
||||
------------- | ------------- | ------------- | -------------
|
||||
**limit** | **int**| |
|
||||
**updatedUntil** | **DateTime**| |
|
||||
**lastCreationDate** | **DateTime**| | [optional]
|
||||
**lastId** | **String**| | [optional]
|
||||
**userId** | **String**| | [optional]
|
||||
|
||||
### Return type
|
||||
|
||||
[**List<AssetResponseDto>**](AssetResponseDto.md)
|
||||
|
||||
### Authorization
|
||||
|
||||
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
|
||||
|
||||
### HTTP request headers
|
||||
|
||||
- **Content-Type**: Not defined
|
||||
- **Accept**: application/json
|
||||
|
||||
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
|
||||
|
||||
# **getDeltaSync**
|
||||
> AssetDeltaSyncResponseDto getDeltaSync(updatedAfter, userIds)
|
||||
|
||||
|
||||
|
||||
### Example
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
// TODO Configure API key authorization: cookie
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
|
||||
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
|
||||
// TODO Configure API key authorization: api_key
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
|
||||
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
|
||||
// TODO Configure HTTP Bearer authorization: bearer
|
||||
// Case 1. Use String Token
|
||||
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
|
||||
// Case 2. Use Function which generate token.
|
||||
// String yourTokenGeneratorFunction() { ... }
|
||||
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
|
||||
|
||||
final api_instance = SyncApi();
|
||||
final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime |
|
||||
final userIds = []; // List<String> |
|
||||
|
||||
try {
|
||||
final result = api_instance.getDeltaSync(updatedAfter, userIds);
|
||||
print(result);
|
||||
} catch (e) {
|
||||
print('Exception when calling SyncApi->getDeltaSync: $e\n');
|
||||
}
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
Name | Type | Description | Notes
|
||||
------------- | ------------- | ------------- | -------------
|
||||
**updatedAfter** | **DateTime**| |
|
||||
**userIds** | [**List<String>**](String.md)| | [default to const []]
|
||||
|
||||
### Return type
|
||||
|
||||
[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md)
|
||||
|
||||
### Authorization
|
||||
|
||||
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
|
||||
|
||||
### HTTP request headers
|
||||
|
||||
- **Content-Type**: Not defined
|
||||
- **Accept**: application/json
|
||||
|
||||
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
|
||||
|
||||
@ -0,0 +1,150 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class SyncApi {
|
||||
SyncApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient;
|
||||
|
||||
final ApiClient apiClient;
|
||||
|
||||
/// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response].
|
||||
/// Parameters:
|
||||
///
|
||||
/// * [int] limit (required):
|
||||
///
|
||||
/// * [DateTime] updatedUntil (required):
|
||||
///
|
||||
/// * [DateTime] lastCreationDate:
|
||||
///
|
||||
/// * [String] lastId:
|
||||
///
|
||||
/// * [String] userId:
|
||||
Future<Response> getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
|
||||
// ignore: prefer_const_declarations
|
||||
final path = r'/sync/full-sync';
|
||||
|
||||
// ignore: prefer_final_locals
|
||||
Object? postBody;
|
||||
|
||||
final queryParams = <QueryParam>[];
|
||||
final headerParams = <String, String>{};
|
||||
final formParams = <String, String>{};
|
||||
|
||||
if (lastCreationDate != null) {
|
||||
queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate));
|
||||
}
|
||||
if (lastId != null) {
|
||||
queryParams.addAll(_queryParams('', 'lastId', lastId));
|
||||
}
|
||||
queryParams.addAll(_queryParams('', 'limit', limit));
|
||||
queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil));
|
||||
if (userId != null) {
|
||||
queryParams.addAll(_queryParams('', 'userId', userId));
|
||||
}
|
||||
|
||||
const contentTypes = <String>[];
|
||||
|
||||
|
||||
return apiClient.invokeAPI(
|
||||
path,
|
||||
'GET',
|
||||
queryParams,
|
||||
postBody,
|
||||
headerParams,
|
||||
formParams,
|
||||
contentTypes.isEmpty ? null : contentTypes.first,
|
||||
);
|
||||
}
|
||||
|
||||
/// Parameters:
|
||||
///
|
||||
/// * [int] limit (required):
|
||||
///
|
||||
/// * [DateTime] updatedUntil (required):
|
||||
///
|
||||
/// * [DateTime] lastCreationDate:
|
||||
///
|
||||
/// * [String] lastId:
|
||||
///
|
||||
/// * [String] userId:
|
||||
Future<List<AssetResponseDto>?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
|
||||
final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil, lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, );
|
||||
if (response.statusCode >= HttpStatus.badRequest) {
|
||||
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
|
||||
}
|
||||
// When a remote server returns no body with a status of 204, we shall not decode it.
|
||||
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
|
||||
// FormatException when trying to decode an empty string.
|
||||
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
|
||||
final responseBody = await _decodeBodyBytes(response);
|
||||
return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
|
||||
.cast<AssetResponseDto>()
|
||||
.toList(growable: false);
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response].
|
||||
/// Parameters:
|
||||
///
|
||||
/// * [DateTime] updatedAfter (required):
|
||||
///
|
||||
/// * [List<String>] userIds (required):
|
||||
Future<Response> getDeltaSyncWithHttpInfo(DateTime updatedAfter, List<String> userIds,) async {
|
||||
// ignore: prefer_const_declarations
|
||||
final path = r'/sync/delta-sync';
|
||||
|
||||
// ignore: prefer_final_locals
|
||||
Object? postBody;
|
||||
|
||||
final queryParams = <QueryParam>[];
|
||||
final headerParams = <String, String>{};
|
||||
final formParams = <String, String>{};
|
||||
|
||||
queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
|
||||
queryParams.addAll(_queryParams('multi', 'userIds', userIds));
|
||||
|
||||
const contentTypes = <String>[];
|
||||
|
||||
|
||||
return apiClient.invokeAPI(
|
||||
path,
|
||||
'GET',
|
||||
queryParams,
|
||||
postBody,
|
||||
headerParams,
|
||||
formParams,
|
||||
contentTypes.isEmpty ? null : contentTypes.first,
|
||||
);
|
||||
}
|
||||
|
||||
/// Parameters:
|
||||
///
|
||||
/// * [DateTime] updatedAfter (required):
|
||||
///
|
||||
/// * [List<String>] userIds (required):
|
||||
Future<AssetDeltaSyncResponseDto?> getDeltaSync(DateTime updatedAfter, List<String> userIds,) async {
|
||||
final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,);
|
||||
if (response.statusCode >= HttpStatus.badRequest) {
|
||||
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
|
||||
}
|
||||
// When a remote server returns no body with a status of 204, we shall not decode it.
|
||||
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
|
||||
// FormatException when trying to decode an empty string.
|
||||
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
|
||||
return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto;
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,116 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
class AssetDeltaSyncResponseDto {
|
||||
/// Returns a new [AssetDeltaSyncResponseDto] instance.
|
||||
AssetDeltaSyncResponseDto({
|
||||
this.deleted = const [],
|
||||
required this.needsFullSync,
|
||||
this.upserted = const [],
|
||||
});
|
||||
|
||||
List<String> deleted;
|
||||
|
||||
bool needsFullSync;
|
||||
|
||||
List<AssetResponseDto> upserted;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncResponseDto &&
|
||||
_deepEquality.equals(other.deleted, deleted) &&
|
||||
other.needsFullSync == needsFullSync &&
|
||||
_deepEquality.equals(other.upserted, upserted);
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(deleted.hashCode) +
|
||||
(needsFullSync.hashCode) +
|
||||
(upserted.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'AssetDeltaSyncResponseDto[deleted=$deleted, needsFullSync=$needsFullSync, upserted=$upserted]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'deleted'] = this.deleted;
|
||||
json[r'needsFullSync'] = this.needsFullSync;
|
||||
json[r'upserted'] = this.upserted;
|
||||
return json;
|
||||
}
|
||||
|
||||
/// Returns a new [AssetDeltaSyncResponseDto] instance and imports its values from
|
||||
/// [value] if it's a [Map], null otherwise.
|
||||
// ignore: prefer_constructors_over_static_methods
|
||||
static AssetDeltaSyncResponseDto? fromJson(dynamic value) {
|
||||
if (value is Map) {
|
||||
final json = value.cast<String, dynamic>();
|
||||
|
||||
return AssetDeltaSyncResponseDto(
|
||||
deleted: json[r'deleted'] is Iterable
|
||||
? (json[r'deleted'] as Iterable).cast<String>().toList(growable: false)
|
||||
: const [],
|
||||
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
|
||||
upserted: AssetResponseDto.listFromJson(json[r'upserted']),
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<AssetDeltaSyncResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <AssetDeltaSyncResponseDto>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = AssetDeltaSyncResponseDto.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
|
||||
static Map<String, AssetDeltaSyncResponseDto> mapFromJson(dynamic json) {
|
||||
final map = <String, AssetDeltaSyncResponseDto>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
|
||||
for (final entry in json.entries) {
|
||||
final value = AssetDeltaSyncResponseDto.fromJson(entry.value);
|
||||
if (value != null) {
|
||||
map[entry.key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// maps a json object with a list of AssetDeltaSyncResponseDto-objects as value to a dart map
|
||||
static Map<String, List<AssetDeltaSyncResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||
final map = <String, List<AssetDeltaSyncResponseDto>>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
// ignore: parameter_assignments
|
||||
json = json.cast<String, dynamic>();
|
||||
for (final entry in json.entries) {
|
||||
map[entry.key] = AssetDeltaSyncResponseDto.listFromJson(entry.value, growable: growable,);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/// The list of required keys that must be present in a JSON.
|
||||
static const requiredKeys = <String>{
|
||||
'deleted',
|
||||
'needsFullSync',
|
||||
'upserted',
|
||||
};
|
||||
}
|
||||
|
||||
@ -0,0 +1,37 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// tests for AssetDeltaSyncResponseDto
|
||||
void main() {
|
||||
// final instance = AssetDeltaSyncResponseDto();
|
||||
|
||||
group('test AssetDeltaSyncResponseDto', () {
|
||||
// List<String> deleted (default value: const [])
|
||||
test('to test the property `deleted`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// bool needsFullSync
|
||||
test('to test the property `needsFullSync`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// List<AssetResponseDto> upserted (default value: const [])
|
||||
test('to test the property `upserted`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
|
||||
/// tests for SyncApi
|
||||
void main() {
|
||||
// final instance = SyncApi();
|
||||
|
||||
group('tests for SyncApi', () {
|
||||
//Future<List<AssetResponseDto>> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime lastCreationDate, String lastId, String userId }) async
|
||||
test('test getAllForUserFullSync', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
//Future<AssetDeltaSyncResponseDto> getDeltaSync(DateTime updatedAfter, List<String> userIds) async
|
||||
test('test getDeltaSync', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
|
||||
import { Auth, Authenticated } from 'src/middleware/auth.guard';
|
||||
import { SyncService } from 'src/services/sync.service';
|
||||
|
||||
@ApiTags('Sync')
|
||||
@Controller('sync')
|
||||
@Authenticated()
|
||||
export class SyncController {
|
||||
constructor(private service: SyncService) {}
|
||||
|
||||
@Get('full-sync')
|
||||
getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
|
||||
return this.service.getAllAssetsForUserFullSync(auth, dto);
|
||||
}
|
||||
|
||||
@Get('delta-sync')
|
||||
getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
|
||||
return this.service.getChangesForDeltaSync(auth, dto);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,38 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsInt, IsPositive } from 'class-validator';
|
||||
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
|
||||
import { ValidateDate, ValidateUUID } from 'src/validation';
|
||||
|
||||
export class AssetFullSyncDto {
|
||||
@ValidateUUID({ optional: true })
|
||||
lastId?: string;
|
||||
|
||||
@ValidateDate({ optional: true })
|
||||
lastCreationDate?: Date;
|
||||
|
||||
@ValidateDate()
|
||||
updatedUntil!: Date;
|
||||
|
||||
@IsInt()
|
||||
@IsPositive()
|
||||
@Type(() => Number)
|
||||
@ApiProperty({ type: 'integer' })
|
||||
limit!: number;
|
||||
|
||||
@ValidateUUID({ optional: true })
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
export class AssetDeltaSyncDto {
|
||||
@ValidateDate()
|
||||
updatedAfter!: Date;
|
||||
@ValidateUUID({ each: true })
|
||||
userIds!: string[];
|
||||
}
|
||||
|
||||
export class AssetDeltaSyncResponseDto {
|
||||
needsFullSync!: boolean;
|
||||
upserted!: AssetResponseDto[];
|
||||
deleted!: string[];
|
||||
}
|
||||
@ -1,14 +1,14 @@
|
||||
import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
|
||||
export const IAuditRepository = 'IAuditRepository';
|
||||
|
||||
export interface AuditSearch {
|
||||
action?: DatabaseAction;
|
||||
entityType?: EntityType;
|
||||
ownerId?: string;
|
||||
userIds: string[];
|
||||
}
|
||||
|
||||
export interface IAuditRepository {
|
||||
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
|
||||
getAfter(since: Date, options: AuditSearch): Promise<string[]>;
|
||||
removeBefore(before: Date): Promise<void>;
|
||||
}
|
||||
|
||||
@ -0,0 +1,101 @@
|
||||
import { mapAsset } from 'src/dtos/asset-response.dto';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { IAccessRepository } from 'src/interfaces/access.interface';
|
||||
import { IAssetRepository } from 'src/interfaces/asset.interface';
|
||||
import { IAuditRepository } from 'src/interfaces/audit.interface';
|
||||
import { IPartnerRepository } from 'src/interfaces/partner.interface';
|
||||
import { SyncService } from 'src/services/sync.service';
|
||||
import { assetStub } from 'test/fixtures/asset.stub';
|
||||
import { authStub } from 'test/fixtures/auth.stub';
|
||||
import { partnerStub } from 'test/fixtures/partner.stub';
|
||||
import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
|
||||
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
|
||||
import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock';
|
||||
import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock';
|
||||
|
||||
const untilDate = new Date(2024);
|
||||
const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true };
|
||||
|
||||
describe(SyncService.name, () => {
|
||||
let sut: SyncService;
|
||||
let accessMock: jest.Mocked<IAccessRepository>;
|
||||
let assetMock: jest.Mocked<IAssetRepository>;
|
||||
let partnerMock: jest.Mocked<IPartnerRepository>;
|
||||
let auditMock: jest.Mocked<IAuditRepository>;
|
||||
|
||||
beforeEach(() => {
|
||||
partnerMock = newPartnerRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
accessMock = newAccessRepositoryMock();
|
||||
auditMock = newAuditRepositoryMock();
|
||||
sut = new SyncService(accessMock, assetMock, partnerMock, auditMock);
|
||||
});
|
||||
|
||||
it('should exist', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getAllAssetsForUserFullSync', () => {
|
||||
it('should return a list of all assets owned by the user', async () => {
|
||||
assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
|
||||
await expect(
|
||||
sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
|
||||
).resolves.toEqual([
|
||||
mapAsset(assetStub.external, mapAssetOpts),
|
||||
mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
|
||||
]);
|
||||
expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
|
||||
ownerId: authStub.user1.user.id,
|
||||
updatedUntil: untilDate,
|
||||
limit: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getChangesForDeltaSync', () => {
|
||||
it('should return a response requiring a full sync when partners are out of sync', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response requiring a full sync when last sync was too long ago', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response requiring a full sync when there are too many changes', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
assetMock.getChangedDeltaSync.mockResolvedValue(
|
||||
Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
|
||||
);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response with changes and deletions', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
|
||||
auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({
|
||||
needsFullSync: false,
|
||||
upserted: [mapAsset(assetStub.image1, mapAssetOpts)],
|
||||
deleted: [assetStub.external.id],
|
||||
});
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,77 @@
|
||||
import { Inject } from '@nestjs/common';
|
||||
import _ from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
|
||||
import { AccessCore, Permission } from 'src/cores/access.core';
|
||||
import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
|
||||
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
import { IAccessRepository } from 'src/interfaces/access.interface';
|
||||
import { IAssetRepository } from 'src/interfaces/asset.interface';
|
||||
import { IAuditRepository } from 'src/interfaces/audit.interface';
|
||||
import { IPartnerRepository } from 'src/interfaces/partner.interface';
|
||||
|
||||
export class SyncService {
|
||||
private access: AccessCore;
|
||||
|
||||
constructor(
|
||||
@Inject(IAccessRepository) accessRepository: IAccessRepository,
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
|
||||
@Inject(IAuditRepository) private auditRepository: IAuditRepository,
|
||||
) {
|
||||
this.access = AccessCore.create(accessRepository);
|
||||
}
|
||||
|
||||
async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
|
||||
const userId = dto.userId || auth.user.id;
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
|
||||
const assets = await this.assetRepository.getAllForUserFullSync({
|
||||
ownerId: userId,
|
||||
lastCreationDate: dto.lastCreationDate,
|
||||
updatedUntil: dto.updatedUntil,
|
||||
lastId: dto.lastId,
|
||||
limit: dto.limit,
|
||||
});
|
||||
const options = { auth, stripMetadata: false, withStack: true };
|
||||
return assets.map((a) => mapAsset(a, options));
|
||||
}
|
||||
|
||||
async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
|
||||
const partner = await this.partnerRepository.getAll(auth.user.id);
|
||||
const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
|
||||
userIds.sort();
|
||||
dto.userIds.sort();
|
||||
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
|
||||
|
||||
if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
|
||||
// app does not have the correct partners synced
|
||||
// or app has not synced in the last 100 days
|
||||
return { needsFullSync: true, deleted: [], upserted: [] };
|
||||
}
|
||||
|
||||
const limit = 10_000;
|
||||
const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
|
||||
|
||||
if (upserted.length === limit) {
|
||||
// too many changes -> do a full sync (paginated) instead
|
||||
return { needsFullSync: true, deleted: [], upserted: [] };
|
||||
}
|
||||
|
||||
const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
|
||||
userIds: userIds,
|
||||
entityType: EntityType.ASSET,
|
||||
action: DatabaseAction.DELETE,
|
||||
});
|
||||
|
||||
const options = { auth, stripMetadata: false, withStack: true };
|
||||
const result = {
|
||||
needsFullSync: false,
|
||||
upserted: upserted.map((a) => mapAsset(a, options)),
|
||||
deleted,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue