mirror of https://github.com/immich-app/immich.git
Added machine learning microservice and object detection (#76)
Alex
GitHub
parent
fe693db84f
commit
dd9c5244fd
@ -1,8 +1,8 @@
|
||||
dev:
|
||||
docker-compose -f ./docker/docker-compose.yml up
|
||||
docker-compose -f ./docker/docker-compose.yml up --remove-orphans
|
||||
|
||||
dev-update:
|
||||
docker-compose -f ./docker/docker-compose.yml up --build -V
|
||||
docker-compose -f ./docker/docker-compose.yml up --build -V --remove-orphans
|
||||
|
||||
dev-scale:
|
||||
docker-compose -f ./docker/docker-compose.yml up --build -V --scale immich_server=3 --remove-orphans
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
node_modules/
|
||||
upload/
|
||||
dist/
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
##################################
|
||||
# DEVELOPMENT
|
||||
##################################
|
||||
FROM node:16-bullseye-slim AS development
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
|
||||
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN npm run build
|
||||
|
||||
#################################
|
||||
# PRODUCTION
|
||||
#################################
|
||||
FROM node:16-bullseye-slim AS production
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG NODE_ENV=production
|
||||
ENV NODE_ENV=${NODE_ENV}
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
|
||||
|
||||
RUN npm install --only=production
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=development /usr/src/app/dist ./dist
|
||||
|
||||
CMD ["node", "dist/main"]
|
||||
@ -1,73 +1,4 @@
|
||||
<p align="center">
|
||||
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
|
||||
</p>
|
||||
|
||||
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
|
||||
[circleci-url]: https://circleci.com/gh/nestjs/nest
|
||||
# Microservices for Immich
|
||||
|
||||
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
|
||||
<p align="center">
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
|
||||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
|
||||
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
|
||||
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
|
||||
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
|
||||
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
|
||||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
|
||||
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
|
||||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
|
||||
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
|
||||
</p>
|
||||
<!--[](https://opencollective.com/nest#backer)
|
||||
[](https://opencollective.com/nest#sponsor)-->
|
||||
|
||||
## Description
|
||||
|
||||
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
$ npm install
|
||||
```
|
||||
|
||||
## Running the app
|
||||
|
||||
```bash
|
||||
# development
|
||||
$ npm run start
|
||||
|
||||
# watch mode
|
||||
$ npm run start:dev
|
||||
|
||||
# production mode
|
||||
$ npm run start:prod
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
# unit tests
|
||||
$ npm run test
|
||||
|
||||
# e2e tests
|
||||
$ npm run test:e2e
|
||||
|
||||
# test coverage
|
||||
$ npm run test:cov
|
||||
```
|
||||
|
||||
## Support
|
||||
|
||||
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
|
||||
|
||||
## Stay in touch
|
||||
|
||||
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
|
||||
- Website - [https://nestjs.com](https://nestjs.com/)
|
||||
- Twitter - [@nestframework](https://twitter.com/nestframework)
|
||||
|
||||
## License
|
||||
|
||||
Nest is [MIT licensed](LICENSE).
|
||||
## Image Classifier
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,12 +0,0 @@
|
||||
import { Controller, Get } from '@nestjs/common';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
@Controller()
|
||||
export class AppController {
|
||||
constructor(private readonly appService: AppService) {}
|
||||
|
||||
@Get()
|
||||
getHello(): string {
|
||||
return this.appService.getHello();
|
||||
}
|
||||
}
|
||||
@ -1,10 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AppController } from './app.controller';
|
||||
import { AppService } from './app.service';
|
||||
import { ImageClassifierModule } from './image-classifier/image-classifier.module';
|
||||
import { databaseConfig } from './config/database.config';
|
||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||
import { ObjectDetectionModule } from './object-detection/object-detection.module';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AppController],
|
||||
providers: [AppService],
|
||||
imports: [
|
||||
TypeOrmModule.forRoot(databaseConfig),
|
||||
ImageClassifierModule,
|
||||
ObjectDetectionModule,
|
||||
],
|
||||
controllers: [],
|
||||
providers: [],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
@Injectable()
|
||||
export class AppService {
|
||||
getHello(): string {
|
||||
console.log('Hello World 123');
|
||||
return 'Hello World!';
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
import { TypeOrmModuleOptions } from '@nestjs/typeorm';
|
||||
|
||||
export const databaseConfig: TypeOrmModuleOptions = {
|
||||
type: 'postgres',
|
||||
host: 'immich_postgres',
|
||||
port: 5432,
|
||||
username: process.env.DB_USERNAME,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_DATABASE_NAME,
|
||||
synchronize: false,
|
||||
};
|
||||
@ -0,0 +1,14 @@
|
||||
import { Body, Controller, Post } from '@nestjs/common';
|
||||
import { ImageClassifierService } from './image-classifier.service';
|
||||
|
||||
@Controller('image-classifier')
|
||||
export class ImageClassifierController {
|
||||
constructor(
|
||||
private readonly imageClassifierService: ImageClassifierService,
|
||||
) {}
|
||||
|
||||
@Post('/tagImage')
|
||||
async tagImage(@Body('thumbnailPath') thumbnailPath: string) {
|
||||
return await this.imageClassifierService.tagImage(thumbnailPath);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ImageClassifierService } from './image-classifier.service';
|
||||
import { ImageClassifierController } from './image-classifier.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [ImageClassifierController],
|
||||
providers: [ImageClassifierService],
|
||||
})
|
||||
export class ImageClassifierModule {}
|
||||
@ -0,0 +1,48 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as mobilenet from '@tensorflow-models/mobilenet';
|
||||
import * as cocoSsd from '@tensorflow-models/coco-ssd';
|
||||
import * as tf from '@tensorflow/tfjs-node';
|
||||
import * as fs from 'fs';
|
||||
|
||||
@Injectable()
|
||||
export class ImageClassifierService {
|
||||
private readonly MOBILENET_VERSION = 2;
|
||||
private readonly MOBILENET_ALPHA = 1.0;
|
||||
|
||||
private mobileNetModel: mobilenet.MobileNet;
|
||||
|
||||
constructor() {
|
||||
Logger.log(
|
||||
`Running Node TensorFlow Version : ${tf.version['tfjs']}`,
|
||||
'ImageClassifier',
|
||||
);
|
||||
mobilenet
|
||||
.load({
|
||||
version: this.MOBILENET_VERSION,
|
||||
alpha: this.MOBILENET_ALPHA,
|
||||
})
|
||||
.then((mobilenetModel) => (this.mobileNetModel = mobilenetModel));
|
||||
}
|
||||
|
||||
async tagImage(thumbnailPath: string) {
|
||||
try {
|
||||
const isExist = fs.existsSync(thumbnailPath);
|
||||
if (isExist) {
|
||||
const tags = [];
|
||||
const image = fs.readFileSync(thumbnailPath);
|
||||
const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
|
||||
const predictions = await this.mobileNetModel.classify(decodedImage);
|
||||
|
||||
for (const prediction of predictions) {
|
||||
if (prediction.probability >= 0.1) {
|
||||
tags.push(...prediction.className.split(',').map((e) => e.trim()));
|
||||
}
|
||||
}
|
||||
|
||||
return tags;
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error reading file ', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,15 +1,10 @@
|
||||
import { NestFactory } from '@nestjs/core';
|
||||
import { AppModule } from './app.module';
|
||||
import { AppService } from './app.service';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.createApplicationContext(AppModule);
|
||||
const app = await NestFactory.create(AppModule);
|
||||
|
||||
const appService = app.get(AppService);
|
||||
|
||||
appService.getHello();
|
||||
|
||||
await app.close();
|
||||
await app.listen(3001);
|
||||
}
|
||||
|
||||
bootstrap();
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
import { Body, Controller, Post } from '@nestjs/common';
|
||||
import { ObjectDetectionService } from './object-detection.service';
|
||||
|
||||
@Controller('object-detection')
|
||||
export class ObjectDetectionController {
|
||||
constructor(
|
||||
private readonly objectDetectionService: ObjectDetectionService,
|
||||
) {}
|
||||
|
||||
@Post('/detectObject')
|
||||
async detectObject(@Body('thumbnailPath') thumbnailPath: string) {
|
||||
return await this.objectDetectionService.detectObject(thumbnailPath);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ObjectDetectionService } from './object-detection.service';
|
||||
import { ObjectDetectionController } from './object-detection.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [ObjectDetectionController],
|
||||
providers: [ObjectDetectionService],
|
||||
})
|
||||
export class ObjectDetectionModule {}
|
||||
@ -0,0 +1,38 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as cocoSsd from '@tensorflow-models/coco-ssd';
|
||||
import * as tf from '@tensorflow/tfjs-node';
|
||||
import * as fs from 'fs';
|
||||
|
||||
@Injectable()
|
||||
export class ObjectDetectionService {
|
||||
private cocoSsdModel: cocoSsd.ObjectDetection;
|
||||
|
||||
constructor() {
|
||||
Logger.log(
|
||||
`Running Node TensorFlow Version : ${tf.version['tfjs']}`,
|
||||
'ObjectDetection',
|
||||
);
|
||||
cocoSsd.load().then((model) => (this.cocoSsdModel = model));
|
||||
}
|
||||
async detectObject(thumbnailPath: string) {
|
||||
try {
|
||||
const isExist = fs.existsSync(thumbnailPath);
|
||||
if (isExist) {
|
||||
const tags = new Set();
|
||||
const image = fs.readFileSync(thumbnailPath);
|
||||
const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
|
||||
const predictions = await this.cocoSsdModel.detect(decodedImage);
|
||||
|
||||
for (const result of predictions) {
|
||||
if (result.score > 0.5) {
|
||||
tags.add(result.class);
|
||||
}
|
||||
}
|
||||
|
||||
return [...tags];
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error reading file ', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,80 @@
|
||||
import 'dart:convert';
|
||||
|
||||
class CuratedObject {
|
||||
final String id;
|
||||
final String object;
|
||||
final String resizePath;
|
||||
final String deviceAssetId;
|
||||
final String deviceId;
|
||||
CuratedObject({
|
||||
required this.id,
|
||||
required this.object,
|
||||
required this.resizePath,
|
||||
required this.deviceAssetId,
|
||||
required this.deviceId,
|
||||
});
|
||||
|
||||
CuratedObject copyWith({
|
||||
String? id,
|
||||
String? object,
|
||||
String? resizePath,
|
||||
String? deviceAssetId,
|
||||
String? deviceId,
|
||||
}) {
|
||||
return CuratedObject(
|
||||
id: id ?? this.id,
|
||||
object: object ?? this.object,
|
||||
resizePath: resizePath ?? this.resizePath,
|
||||
deviceAssetId: deviceAssetId ?? this.deviceAssetId,
|
||||
deviceId: deviceId ?? this.deviceId,
|
||||
);
|
||||
}
|
||||
|
||||
Map<String, dynamic> toMap() {
|
||||
final result = <String, dynamic>{};
|
||||
|
||||
result.addAll({'id': id});
|
||||
result.addAll({'object': object});
|
||||
result.addAll({'resizePath': resizePath});
|
||||
result.addAll({'deviceAssetId': deviceAssetId});
|
||||
result.addAll({'deviceId': deviceId});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
factory CuratedObject.fromMap(Map<String, dynamic> map) {
|
||||
return CuratedObject(
|
||||
id: map['id'] ?? '',
|
||||
object: map['object'] ?? '',
|
||||
resizePath: map['resizePath'] ?? '',
|
||||
deviceAssetId: map['deviceAssetId'] ?? '',
|
||||
deviceId: map['deviceId'] ?? '',
|
||||
);
|
||||
}
|
||||
|
||||
String toJson() => json.encode(toMap());
|
||||
|
||||
factory CuratedObject.fromJson(String source) => CuratedObject.fromMap(json.decode(source));
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return 'CuratedObject(id: $id, object: $object, resizePath: $resizePath, deviceAssetId: $deviceAssetId, deviceId: $deviceId)';
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
if (identical(this, other)) return true;
|
||||
|
||||
return other is CuratedObject &&
|
||||
other.id == id &&
|
||||
other.object == object &&
|
||||
other.resizePath == resizePath &&
|
||||
other.deviceAssetId == deviceAssetId &&
|
||||
other.deviceId == deviceId;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
return id.hashCode ^ object.hashCode ^ resizePath.hashCode ^ deviceAssetId.hashCode ^ deviceId.hashCode;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
extension StringExtension on String {
|
||||
String capitalizeFirstLetter() {
|
||||
return "${this[0].toUpperCase()}${substring(1).toLowerCase()}";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||
|
||||
export class AddObjectColumnToSmartInfo1648317474768
|
||||
implements MigrationInterface
|
||||
{
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE smart_info
|
||||
ADD COLUMN objects text[];
|
||||
|
||||
`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`
|
||||
ALTER TABLE smart_info
|
||||
DROP COLUMN objects;
|
||||
`);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue