sascha
/
immich
mirror of https://github.com/immich-app/immich.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(server): preserve caption fields and extract mediainfo for video (#505) 

* feat(server): preserve caption fields and extract mediainfo for video

* Fixed Geocoding missing info leads to fail EXIF extraction for the whole file

Co-authored-by: Alex <alex.tran1502@gmail.com>
pull/508/head
Thanh Pham 2022-08-21 06:31:37 +07:00 committed by GitHub
parent 013a0f8324
commit 9c30d58b10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 195 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
server/apps/microservices/src/processors/metadata-extraction.processor.ts
Unescape Escape View File

@ -85,9 +85,21 @@ export class MetadataExtractionProcessor {
const res: [] = geoCodeInfo.body['features'];
const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
let city = '';
let state = '';
let country = '';
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
}
newExif.city = city || null;
newExif.state = state || null;
@ -114,9 +126,21 @@ export class MetadataExtractionProcessor {
const res: [] = geoCodeInfo.body['features'];
const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
let city = '';
let state = '';
let country = '';
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
}
await this.exifRepository.update({ id: exif.id }, { city, state, country });
}
@ -168,31 +192,126 @@ export class MetadataExtractionProcessor {
async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
const { asset } = job.data;
ffmpeg.ffprobe(asset.originalPath, async (err, data) => {
if (!err) {
let durationString = asset.duration;
let createdAt = asset.createdAt;
try {
const data = await new Promise<ffmpeg.FfprobeData>((resolve, reject) =>
ffmpeg.ffprobe(asset.originalPath, (err, data) => {
if (err) return reject(err);
return resolve(data);
}),
);
let durationString = asset.duration;
let createdAt = asset.createdAt;
if (data.format.duration) {
durationString = this.extractDuration(data.format.duration);
}
const videoTags = data.format.tags;
if (videoTags) {
if (videoTags['com.apple.quicktime.creationdate']) {
createdAt = String(videoTags['com.apple.quicktime.creationdate']);
} else if (videoTags['creation_time']) {
createdAt = String(videoTags['creation_time']);
} else {
createdAt = asset.createdAt;
}
} else {
createdAt = asset.createdAt;
}
if (data.format.duration) {
durationString = this.extractDuration(data.format.duration);
const newExif = new ExifEntity();
newExif.assetId = asset.id;
newExif.description = '';
newExif.fileSizeInByte = data.format.size || null;
newExif.dateTimeOriginal = createdAt ? new Date(createdAt) : null;
newExif.modifyDate = null;
newExif.latitude = null;
newExif.longitude = null;
newExif.city = null;
newExif.state = null;
newExif.country = null;
newExif.fps = null;
if (videoTags && videoTags['location']) {
const location = videoTags['location'] as string;
const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
const match = location.match(locationRegex);
if (match?.length === 3) {
newExif.latitude = parseFloat(match[0]);
newExif.longitude = parseFloat(match[1]);
}
} else if (videoTags && videoTags['com.apple.quicktime.location.ISO6709']) {
const location = videoTags['com.apple.quicktime.location.ISO6709'] as string;
const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
const match = location.match(locationRegex);
if (match?.length === 4) {
newExif.latitude = parseFloat(match[1]);
newExif.longitude = parseFloat(match[2]);
}
}
const videoTags = data.format.tags;
if (videoTags) {
if (videoTags['com.apple.quicktime.creationdate']) {
createdAt = String(videoTags['com.apple.quicktime.creationdate']);
} else if (videoTags['creation_time']) {
createdAt = String(videoTags['creation_time']);
// Reverse GeoCoding
if (this.geocodingClient && newExif.longitude && newExif.latitude) {
const geoCodeInfo: MapiResponse = await this.geocodingClient
.reverseGeocode({
query: [newExif.longitude, newExif.latitude],
types: ['country', 'region', 'place'],
})
.send();
const res: [] = geoCodeInfo.body['features'];
let city = '';
let state = '';
let country = '';
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
}
if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
}
newExif.city = city || null;
newExif.state = state || null;
newExif.country = country || null;
}
for (const stream of data.streams) {
if (stream.codec_type === 'video') {
newExif.exifImageWidth = stream.width || null;
newExif.exifImageHeight = stream.height || null;
if (typeof stream.rotation === 'string') {
newExif.orientation = stream.rotation;
} else if (typeof stream.rotation === 'number') {
newExif.orientation = `${stream.rotation}`;
} else {
createdAt = asset.createdAt;
newExif.orientation = null;
}
} else {
createdAt = asset.createdAt;
}
await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
if (stream.r_frame_rate) {
let fpsParts = stream.r_frame_rate.split('/');
if (fpsParts.length === 2) {
newExif.fps = Math.round(parseInt(fpsParts[0]) / parseInt(fpsParts[1]));
}
}
}
}
});
await this.exifRepository.save(newExif);
await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
} catch (err) {
// do nothing
console.log('Error in video metadata extraction', err);
}
}
private extractDuration(duration: number) {
@ -202,8 +321,6 @@ export class MetadataExtractionProcessor {
const minutes = Math.floor((videoDurationInSecond - hours * 3600) / 60);
const seconds = videoDurationInSecond - hours * 3600 - minutes * 60;
return `${hours}:${minutes < 10 ? '0' + minutes.toString() : minutes}:${
seconds < 10 ? '0' + seconds.toString() : seconds
}.000000`;
return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.000000`;
}
}

55
server/libs/database/src/entities/exif.entity.ts
Unescape Escape View File

@ -13,14 +13,9 @@ export class ExifEntity {
@Column({ type: 'uuid' })
assetId!: string;
@Column({ type: 'varchar', nullable: true })
make!: string | null;
@Column({ type: 'varchar', nullable: true })
model!: string | null;
@Column({ type: 'varchar', nullable: true })
imageName!: string | null;
/* General info */
@Column({ type: 'text', nullable: true, default: '' })
description!: string; // or caption
@Column({ type: 'integer', nullable: true })
exifImageWidth!: number | null;
@ -40,6 +35,31 @@ export class ExifEntity {
@Column({ type: 'timestamptz', nullable: true })
modifyDate!: Date | null;
@Column({ type: 'float', nullable: true })
latitude!: number | null;
@Column({ type: 'float', nullable: true })
longitude!: number | null;
@Column({ type: 'varchar', nullable: true })
city!: string | null;
@Column({ type: 'varchar', nullable: true })
state!: string | null;
@Column({ type: 'varchar', nullable: true })
country!: string | null;
/* Image info */
@Column({ type: 'varchar', nullable: true })
make!: string | null;
@Column({ type: 'varchar', nullable: true })
model!: string | null;
@Column({ type: 'varchar', nullable: true })
imageName!: string | null;
@Column({ type: 'varchar', nullable: true })
lensModel!: string | null;
@ -55,24 +75,13 @@ export class ExifEntity {
@Column({ type: 'float', nullable: true })
exposureTime!: number | null;
@Column({ type: 'float', nullable: true })
latitude!: number | null;
@Column({ type: 'float', nullable: true })
longitude!: number | null;
@Column({ type: 'varchar', nullable: true })
city!: string | null;
@Column({ type: 'varchar', nullable: true })
state!: string | null;
@Column({ type: 'varchar', nullable: true })
country!: string | null;
/* Video info */
@Column({ type: 'float8', nullable: true })
fps?: number | null;
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
asset?: ExifEntity;
asset?: AssetEntity;
@Index('exif_text_searchable', { synchronize: false })
@Column({

2
server/libs/database/src/entities/smart-info.entity.ts
Unescape Escape View File

@ -18,5 +18,5 @@ export class SmartInfoEntity {
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
asset?: SmartInfoEntity;
asset?: AssetEntity;
}

18
server/libs/database/src/migrations/1661011331242-AddCaption.ts
Unescape Escape View File

@ -0,0 +1,18 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddCaption1661011331242 implements MigrationInterface {
name = 'AddCaption1661011331242'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "exif" ADD "description" text DEFAULT ''`);
await queryRunner.query(`ALTER TABLE "exif" ADD "fps" double precision`);
// await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" SET NOT NULL`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
// await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" DROP NOT NULL`);
await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "fps"`);
await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "description"`);
}
}