mirror of https://github.com/immich-app/immich.git
WIP refactor container and queuing system (#206)
* refactor microservices to machine-learning * Update tGithub issue template with correct task syntax * Added microservices container * Communicate between service based on queue system * added dependency * Fixed problem with having to import BullQueue into the individual service * Added todo * refactor server into monorepo with microservices * refactor database and entity to library * added simple migration * Move migrations and database config to library * Migration works in library * Cosmetic change in logging message * added user dto * Fixed issue with testing not able to find the shared library * Clean up library mapping path * Added webp generator to microservices * Update Github Action build latest * Fixed issue NPM cannot install due to conflict witl Bull Queue * format project with prettier * Modified docker-compose file * Add GH Action for Staging build: * Fixed GH action job name * Modified GH Action to only build & push latest when pushing to main * Added Test 2e2 Github Action * Added Test 2e2 Github Action * Implemented microservice to extract exif * Added cronjob to scan and generate webp thumbnail at midnight * Refactor to ireduce hit time to database when running microservices * Added error handling to asset services that handle read file from disk * Added video transcoding queue to process one video at a time * Fixed loading spinner on web while loading covering the info panel * Add mechanism to show new release announcement to web and mobile app (#209) * Added changelog page * Fixed issues based on PR comments * Fixed issue with video transcoding run on the server * Change entry point content for backward combatibility when starting up server * Added announcement box * Added error handling to failed silently when the app version checking is not able to make the request to GITHUB * Added new version announcement overlay * Update message * Added messages * Added logic to check and show announcement * Add method to handle saving new version * Added button to dimiss the acknowledge message * Up version for deployment to the app storepull/219/head v1.11.0_17-dev
Alex
GitHub
parent
397f8c70b4
commit
a8220172f8
@ -0,0 +1,95 @@
|
||||
name: Build and Push Docker Image - Staging
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
# This image include both the server and microservices - the two containers can be slitted into separated
|
||||
# service with its coressponding entry file.
|
||||
build_and_push_server_monorepo_staging:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2.0.0
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2.0.0
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and push Immich Mono Repo
|
||||
uses: docker/build-push-action@v3.0.0
|
||||
with:
|
||||
context: ./server
|
||||
file: ./server/Dockerfile
|
||||
platforms: linux/arm/v7,linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name == 'pull_request' }}
|
||||
tags: |
|
||||
altran1502/immich-server:staging
|
||||
|
||||
build_and_push_machine_learning_staging:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2.0.0
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2.0.0
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and Push Machine Learning
|
||||
uses: docker/build-push-action@v3.0.0
|
||||
with:
|
||||
context: ./machine-learning
|
||||
file: ./machine-learning/Dockerfile
|
||||
platforms: linux/arm/v7,linux/amd64
|
||||
push: ${{ github.event_name == 'pull_request' }}
|
||||
tags: |
|
||||
altran1502/immich-machine-learning:staging
|
||||
|
||||
build_and_push_web_staging:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2.0.0
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2.0.0
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and Push Web
|
||||
uses: docker/build-push-action@v3.0.0
|
||||
with:
|
||||
context: ./web
|
||||
file: ./web/Dockerfile
|
||||
platforms: linux/arm/v7,linux/amd64,linux/arm64
|
||||
target: prod
|
||||
push: ${{ github.event_name == 'pull_request' }}
|
||||
tags: |
|
||||
altran1502/immich-web:staging
|
||||
@ -1 +0,0 @@
|
||||
devenv/
|
||||
@ -1,3 +0,0 @@
|
||||
__pycache__/
|
||||
devenv/
|
||||
app/upload
|
||||
@ -1,25 +0,0 @@
|
||||
## GPU Build
|
||||
# FROM tensorflow/tensorflow:latest-gpu as gpu
|
||||
|
||||
# WORKDIR /code
|
||||
|
||||
# COPY ./requirements.txt /code/requirements.txt
|
||||
|
||||
# RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
|
||||
|
||||
# COPY ./app /code/app
|
||||
|
||||
|
||||
## CPU BUILD
|
||||
FROM python:3.8 as cpu
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install ffmpeg libsm6 libxext6 -y
|
||||
|
||||
WORKDIR /code
|
||||
|
||||
COPY ./requirements.txt /code/requirements.txt
|
||||
|
||||
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
|
||||
|
||||
COPY ./app /code/app
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 193 KiB |
@ -1,37 +0,0 @@
|
||||
from tensorflow.keras.applications import InceptionV3
|
||||
from tensorflow.keras.applications.inception_v3 import preprocess_input, decode_predictions
|
||||
from tensorflow.keras.preprocessing import image
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
import cv2
|
||||
IMG_SIZE = 299
|
||||
PREDICTION_MODEL = InceptionV3(weights='imagenet')
|
||||
|
||||
|
||||
def classify_image(image_path: str):
|
||||
img_path = f'./app/{image_path}'
|
||||
# img = image.load_img(img_path, target_size=(IMG_SIZE, IMG_SIZE))
|
||||
|
||||
target_image = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
|
||||
resized_target_image = cv2.resize(target_image, (IMG_SIZE, IMG_SIZE))
|
||||
|
||||
x = image.img_to_array(resized_target_image)
|
||||
x = np.expand_dims(x, axis=0)
|
||||
x = preprocess_input(x)
|
||||
|
||||
preds = PREDICTION_MODEL.predict(x)
|
||||
result = decode_predictions(preds, top=3)[0]
|
||||
payload = []
|
||||
for _, value, _ in result:
|
||||
payload.append(value)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def warm_up():
|
||||
img_path = f'./app/test.png'
|
||||
img = image.load_img(img_path, target_size=(IMG_SIZE, IMG_SIZE))
|
||||
x = image.img_to_array(img)
|
||||
x = np.expand_dims(x, axis=0)
|
||||
x = preprocess_input(x)
|
||||
PREDICTION_MODEL.predict(x)
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,46 +0,0 @@
|
||||
from pydantic import BaseModel
|
||||
from fastapi import FastAPI
|
||||
|
||||
from .object_detection import object_detection
|
||||
from .image_classifier import image_classifier
|
||||
|
||||
from tf2_yolov4.anchors import YOLOV4_ANCHORS
|
||||
from tf2_yolov4.model import YOLOv4
|
||||
|
||||
|
||||
HEIGHT, WIDTH = (640, 960)
|
||||
|
||||
# Warm up model
|
||||
image_classifier.warm_up()
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
class TagImagePayload(BaseModel):
|
||||
thumbnail_path: str
|
||||
|
||||
|
||||
@app.post("/tagImage")
|
||||
async def post_root(payload: TagImagePayload):
|
||||
image_path = payload.thumbnail_path
|
||||
|
||||
if image_path[0] == '.':
|
||||
image_path = image_path[2:]
|
||||
|
||||
return image_classifier.classify_image(image_path=image_path)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def test():
|
||||
|
||||
object_detection.run_detection()
|
||||
# image = tf.io.read_file("./app/cars.jpg")
|
||||
# image = tf.image.decode_image(image)
|
||||
# image = tf.image.resize(image, (HEIGHT, WIDTH))
|
||||
# images = tf.expand_dims(image, axis=0) / 255.0
|
||||
|
||||
# model = YOLOv4(
|
||||
# (HEIGHT, WIDTH, 3),
|
||||
# 80,
|
||||
# YOLOV4_ANCHORS,
|
||||
# "darknet",
|
||||
# )
|
||||
@ -1,4 +0,0 @@
|
||||
|
||||
|
||||
def run_detection():
|
||||
print("run detection")
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 345 KiB |
@ -1,8 +0,0 @@
|
||||
opencv-python==4.5.5.64
|
||||
fastapi>=0.68.0,<0.69.0
|
||||
pydantic>=1.8.0,<2.0.0
|
||||
uvicorn>=0.15.0,<0.16.0
|
||||
tensorflow==2.8.0
|
||||
numpy==1.22.2
|
||||
pillow==9.0.1
|
||||
tf2_yolov4==0.1.0
|
||||
@ -0,0 +1 @@
|
||||
* Added announcement pop-up when a new released is pushed out in Github.
|
||||
@ -0,0 +1,57 @@
|
||||
import 'package:dio/dio.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:hive_flutter/hive_flutter.dart';
|
||||
import 'package:hooks_riverpod/hooks_riverpod.dart';
|
||||
import 'package:immich_mobile/constants/hive_box.dart';
|
||||
import 'package:immich_mobile/shared/views/version_announcement_overlay.dart';
|
||||
|
||||
class ReleaseInfoNotifier extends StateNotifier<String> {
|
||||
ReleaseInfoNotifier() : super("");
|
||||
|
||||
void checkGithubReleaseInfo() async {
|
||||
var dio = Dio();
|
||||
var box = Hive.box(hiveGithubReleaseInfoBox);
|
||||
|
||||
try {
|
||||
String? localReleaseVersion = box.get(githubReleaseInfoKey);
|
||||
|
||||
Response res = await dio.get(
|
||||
"https://api.github.com/repos/alextran1502/immich/releases/latest",
|
||||
options: Options(
|
||||
headers: {"Accept": "application/vnd.github.v3+json"},
|
||||
),
|
||||
);
|
||||
|
||||
if (res.statusCode == 200) {
|
||||
String latestTagVersion = res.data["tag_name"];
|
||||
state = latestTagVersion;
|
||||
|
||||
debugPrint("Local release version $localReleaseVersion");
|
||||
debugPrint("Remote release veresion $latestTagVersion");
|
||||
|
||||
if (localReleaseVersion == null && latestTagVersion.isNotEmpty) {
|
||||
VersionAnnouncementOverlayController.appLoader.show();
|
||||
return;
|
||||
}
|
||||
|
||||
if (latestTagVersion.isNotEmpty && localReleaseVersion != latestTagVersion) {
|
||||
VersionAnnouncementOverlayController.appLoader.show();
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
debugPrint("Error gettting latest release version");
|
||||
|
||||
state = "";
|
||||
}
|
||||
}
|
||||
|
||||
void acknowledgeNewVersion() {
|
||||
var box = Hive.box(hiveGithubReleaseInfoBox);
|
||||
|
||||
box.put(githubReleaseInfoKey, state);
|
||||
VersionAnnouncementOverlayController.appLoader.hide();
|
||||
}
|
||||
}
|
||||
|
||||
final releaseInfoProvider = StateNotifierProvider<ReleaseInfoNotifier, String>((ref) => ReleaseInfoNotifier());
|
||||
@ -0,0 +1,133 @@
|
||||
import 'package:flutter/gestures.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:hooks_riverpod/hooks_riverpod.dart';
|
||||
import 'package:immich_mobile/shared/providers/release_info.provider.dart';
|
||||
import 'package:url_launcher/url_launcher.dart';
|
||||
|
||||
class VersionAnnouncementOverlay extends HookConsumerWidget {
|
||||
const VersionAnnouncementOverlay({
|
||||
Key? key,
|
||||
}) : super(key: key);
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context, WidgetRef ref) {
|
||||
void goToReleaseNote() async {
|
||||
final Uri _url = Uri.parse('https://github.com/alextran1502/immich/releases/latest');
|
||||
await launchUrl(_url);
|
||||
}
|
||||
|
||||
void onAcknowledgeTapped() {
|
||||
ref.watch(releaseInfoProvider.notifier).acknowledgeNewVersion();
|
||||
}
|
||||
|
||||
return ValueListenableBuilder<bool>(
|
||||
valueListenable: VersionAnnouncementOverlayController.appLoader.loaderShowingNotifier,
|
||||
builder: (context, shouldShow, child) {
|
||||
if (shouldShow) {
|
||||
return Scaffold(
|
||||
backgroundColor: Colors.black38,
|
||||
body: Center(
|
||||
child: ConstrainedBox(
|
||||
constraints: const BoxConstraints(maxWidth: 307),
|
||||
child: Wrap(
|
||||
children: [
|
||||
Card(
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.all(30.0),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
const Text(
|
||||
"New Server Version Available 🎉",
|
||||
style: TextStyle(
|
||||
fontSize: 16,
|
||||
fontFamily: 'WorkSans',
|
||||
fontWeight: FontWeight.bold,
|
||||
color: Colors.indigo,
|
||||
),
|
||||
),
|
||||
Padding(
|
||||
padding: const EdgeInsets.only(top: 16.0),
|
||||
child: RichText(
|
||||
text: TextSpan(
|
||||
style: const TextStyle(
|
||||
fontSize: 14, fontFamily: 'WorkSans', color: Colors.black87, height: 1.2),
|
||||
children: <TextSpan>[
|
||||
const TextSpan(
|
||||
text: 'Hi friend, there is a new release of',
|
||||
),
|
||||
const TextSpan(
|
||||
text: ' Immich ',
|
||||
style: TextStyle(
|
||||
fontFamily: "SnowBurstOne",
|
||||
color: Colors.indigo,
|
||||
fontWeight: FontWeight.bold,
|
||||
),
|
||||
),
|
||||
const TextSpan(
|
||||
text: "please take your time to visit the ",
|
||||
),
|
||||
TextSpan(
|
||||
text: "release note",
|
||||
style: const TextStyle(
|
||||
decoration: TextDecoration.underline,
|
||||
),
|
||||
recognizer: TapGestureRecognizer()..onTap = goToReleaseNote,
|
||||
),
|
||||
const TextSpan(
|
||||
text:
|
||||
" and ensure your docker-compose and .env setup is up-to-date to prevent any misconfigurations, especially if you use WatchTower or any mechanism that handles updating your server application automatically.",
|
||||
)
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
Padding(
|
||||
padding: const EdgeInsets.only(top: 16.0),
|
||||
child: ElevatedButton(
|
||||
style: ElevatedButton.styleFrom(
|
||||
shape: const StadiumBorder(),
|
||||
visualDensity: VisualDensity.standard,
|
||||
primary: Colors.indigo,
|
||||
onPrimary: Colors.grey[50],
|
||||
elevation: 2,
|
||||
padding: const EdgeInsets.symmetric(vertical: 10, horizontal: 25),
|
||||
),
|
||||
onPressed: onAcknowledgeTapped,
|
||||
child: const Text(
|
||||
"Acknowledge",
|
||||
style: TextStyle(
|
||||
fontSize: 14,
|
||||
),
|
||||
)),
|
||||
)
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
return Container();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class VersionAnnouncementOverlayController {
|
||||
static final VersionAnnouncementOverlayController appLoader = VersionAnnouncementOverlayController();
|
||||
ValueNotifier<bool> loaderShowingNotifier = ValueNotifier(false);
|
||||
ValueNotifier<String> loaderTextNotifier = ValueNotifier('error message');
|
||||
|
||||
void show() {
|
||||
loaderShowingNotifier.value = true;
|
||||
}
|
||||
|
||||
void hide() {
|
||||
loaderShowingNotifier.value = false;
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
import { IsNotEmpty, IsOptional } from 'class-validator';
|
||||
import { AssetType } from '../entities/asset.entity';
|
||||
import { AssetType } from '@app/database/entities/asset.entity';
|
||||
|
||||
export class CreateAssetDto {
|
||||
@IsNotEmpty()
|
||||
@ -1,4 +1,4 @@
|
||||
import { AssetEntity } from '../entities/asset.entity';
|
||||
import { AssetEntity } from '@app/database/entities/asset.entity';
|
||||
|
||||
export class GetAllAssetReponseDto {
|
||||
data: Array<{ date: string; assets: Array<AssetEntity> }>;
|
||||
@ -1,7 +1,7 @@
|
||||
import { BadRequestException, Injectable, InternalServerErrorException, Logger } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { UserEntity } from '../user/entities/user.entity';
|
||||
import { UserEntity } from '@app/database/entities/user.entity';
|
||||
import { LoginCredentialDto } from './dto/login-credential.dto';
|
||||
import { ImmichJwtService } from '../../modules/immich-jwt/immich-jwt.service';
|
||||
import { JwtPayloadDto } from './dto/jwt-payload.dto';
|
||||
@ -1,5 +1,5 @@
|
||||
import { IsNotEmpty, IsOptional } from 'class-validator';
|
||||
import { DeviceType } from '../entities/device-info.entity';
|
||||
import { DeviceType } from '@app/database/entities/device-info.entity';
|
||||
|
||||
export class CreateDeviceInfoDto {
|
||||
@IsNotEmpty()
|
||||
@ -1,6 +1,6 @@
|
||||
import { PartialType } from '@nestjs/mapped-types';
|
||||
import { IsOptional } from 'class-validator';
|
||||
import { DeviceType } from '../entities/device-info.entity';
|
||||
import { DeviceType } from '@app/database/entities/device-info.entity';
|
||||
import { CreateDeviceInfoDto } from './create-device-info.dto';
|
||||
|
||||
export class UpdateDeviceInfoDto extends PartialType(CreateDeviceInfoDto) {}
|
||||
@ -1,5 +1,5 @@
|
||||
import { IsNotEmpty } from 'class-validator';
|
||||
import { AssetEntity } from '../../asset/entities/asset.entity';
|
||||
import { AssetEntity } from '@app/database/entities/asset.entity';
|
||||
|
||||
export class AddAssetsDto {
|
||||
@IsNotEmpty()
|
||||
@ -1,5 +1,5 @@
|
||||
import { IsNotEmpty, IsOptional } from 'class-validator';
|
||||
import { AssetEntity } from '../../asset/entities/asset.entity';
|
||||
import { AssetEntity } from '@app/database/entities/asset.entity';
|
||||
|
||||
export class CreateSharedAlbumDto {
|
||||
@IsNotEmpty()
|
||||
@ -1,4 +1,4 @@
|
||||
import { UserEntity } from '../entities/user.entity';
|
||||
import { UserEntity } from '../../../../../../libs/database/src/entities/user.entity';
|
||||
|
||||
export interface User {
|
||||
id: string;
|
||||
@ -1,15 +1,13 @@
|
||||
import { Controller, Get, Res, Headers } from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
|
||||
@Controller()
|
||||
|
||||
export class AppController {
|
||||
constructor() { }
|
||||
constructor() {}
|
||||
|
||||
@Get()
|
||||
async redirectToWebpage(@Res({ passthrough: true }) res: Response, @Headers() headers) {
|
||||
const host = headers.host;
|
||||
|
||||
return res.redirect(`http://${host}:2285`)
|
||||
return res.redirect(`http://${host}:2285`);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue