feat(web): support 360 video (equirectangular) (#8762)

* [web]: support 360 video

* lint

* lint

* fix typing

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

web/package-lock.json

@@ -12,6 +12,8 @@
         "@immich/sdk": "file:../open-api/typescript-sdk",
         "@mdi/js": "^7.4.47",
         "@photo-sphere-viewer/core": "^5.7.1",
+        "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2",
+        "@photo-sphere-viewer/video-plugin": "^5.7.2",
         "@zoom-image/svelte": "^0.2.6",
         "buffer": "^6.0.3",
         "copy-image-clipboard": "^2.1.2",
@@ -1590,6 +1592,22 @@
         "three": "^0.161.0"
       }
     },
+    "node_modules/@photo-sphere-viewer/equirectangular-video-adapter": {
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/equirectangular-video-adapter/-/equirectangular-video-adapter-5.7.2.tgz",
+      "integrity": "sha512-cAaot52nPqa2p77Xp1humRvuxRIa8cqbZ/XRhA8kBToFLT1Ugh9YBcDD7pM/358JtAjicUbLpT7Ioap9iEigxQ==",
+      "peerDependencies": {
+        "@photo-sphere-viewer/core": "5.7.2"
+      }
+    },
+    "node_modules/@photo-sphere-viewer/video-plugin": {
+      "version": "5.7.2",
+      "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.7.2.tgz",
+      "integrity": "sha512-vrPV9RCr4HsYiORkto1unDPeUkbN2kbyogvNUoLiQ78M4xkPOqoKxtfxCxTYoM+7gECwNL9VTF81+okck498qA==",
+      "peerDependencies": {
+        "@photo-sphere-viewer/core": "5.7.2"
+      }
+    },
     "node_modules/@polka/url": {
       "version": "1.0.0-next.24",
       "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz",

web/package.json

@@ -61,6 +61,8 @@
     "@immich/sdk": "file:../open-api/typescript-sdk",
     "@mdi/js": "^7.4.47",
     "@photo-sphere-viewer/core": "^5.7.1",
+    "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2",
+    "@photo-sphere-viewer/video-plugin": "^5.7.2",
     "@zoom-image/svelte": "^0.2.6",
     "buffer": "^6.0.3",
     "copy-image-clipboard": "^2.1.2",

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -50,7 +50,7 @@
   import PanoramaViewer from './panorama-viewer.svelte';
   import PhotoViewer from './photo-viewer.svelte';
   import SlideshowBar from './slideshow-bar.svelte';
-  import VideoViewer from './video-viewer.svelte';
+  import VideoViewer from './video-wrapper-viewer.svelte';
 
   export let assetStore: AssetStore | null = null;
   export let asset: AssetResponseDto;
@@ -622,6 +622,7 @@
           {:else}
             <VideoViewer
               assetId={previewStackedAsset.id}
+              projectionType={previewStackedAsset.exifInfo?.projectionType}
               on:close={closeViewer}
               on:onVideoEnded={() => navigateAsset()}
               on:onVideoStarted={handleVideoStarted}
@@ -642,6 +643,7 @@
             {#if shouldPlayMotionPhoto && asset.livePhotoVideoId}
               <VideoViewer
                 assetId={asset.livePhotoVideoId}
+                projectionType={asset.exifInfo?.projectionType}
                 on:close={closeViewer}
                 on:onVideoEnded={() => (shouldPlayMotionPhoto = false)}
               />
@@ -655,6 +657,7 @@
           {:else}
             <VideoViewer
               assetId={asset.id}
+              projectionType={asset.exifInfo?.projectionType}
               on:close={closeViewer}
               on:onVideoEnded={() => navigateAsset()}
               on:onVideoStarted={handleVideoStarted}

web/src/lib/components/asset-viewer/panorama-viewer.svelte

@@ -1,22 +1,39 @@
 <script lang="ts">
-  import { serveFile, type AssetResponseDto } from '@immich/sdk';
+  import { serveFile, type AssetResponseDto, AssetTypeEnum } from '@immich/sdk';
   import { fade } from 'svelte/transition';
   import LoadingSpinner from '../shared-components/loading-spinner.svelte';
   import { getKey } from '$lib/utils';
-  export let asset: AssetResponseDto;
+  import type { AdapterConstructor, PluginConstructor } from '@photo-sphere-viewer/core';
+  export let asset: Pick<AssetResponseDto, 'id' | 'type'>;
+
+  const photoSphereConfigs =
+    asset.type === AssetTypeEnum.Video
+      ? ([
+          import('@photo-sphere-viewer/equirectangular-video-adapter').then(
+            ({ EquirectangularVideoAdapter }) => EquirectangularVideoAdapter,
+          ),
+          import('@photo-sphere-viewer/video-plugin').then(({ VideoPlugin }) => [VideoPlugin]),
+          true,
+          import('@photo-sphere-viewer/video-plugin/index.css'),
+        ] as [PromiseLike<AdapterConstructor>, Promise<PluginConstructor[]>, true, unknown])
+      : ([undefined, [], false] as [undefined, [], false]);
 
   const loadAssetData = async () => {
     const data = await serveFile({ id: asset.id, isWeb: false, isThumb: false, key: getKey() });
-    return URL.createObjectURL(data);
+    const url = URL.createObjectURL(data);
+    if (asset.type === AssetTypeEnum.Video) {
+      return { source: url };
+    }
+    return url;
   };
 </script>
 
 <div transition:fade={{ duration: 150 }} class="flex h-full select-none place-content-center place-items-center">
   <!-- the photo sphere viewer is quite large, so lazy load it in parallel with loading the data -->
-  {#await Promise.all([loadAssetData(), import('./photo-sphere-viewer-adapter.svelte')])}
+  {#await Promise.all([loadAssetData(), import('./photo-sphere-viewer-adapter.svelte'), ...photoSphereConfigs])}
     <LoadingSpinner />
-  {:then [data, module]}
+  {:then [data, module, adapter, plugins, navbar]}
-    <svelte:component this={module.default} panorama={data} />
+    <svelte:component this={module.default} panorama={data} plugins={plugins ?? undefined} {navbar} {adapter} />
   {:catch}
     Failed to load asset
   {/await}

web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte

@@ -1,17 +1,32 @@
 <script lang="ts">
-  import { Viewer } from '@photo-sphere-viewer/core';
+  import {
+    Viewer,
+    EquirectangularAdapter,
+    type PluginConstructor,
+    type AdapterConstructor,
+  } from '@photo-sphere-viewer/core';
   import '@photo-sphere-viewer/core/index.css';
   import { onDestroy, onMount } from 'svelte';
 
-  export let panorama: string;
+  export let panorama: string | { source: string };
+  export let adapter: AdapterConstructor | [AdapterConstructor, unknown] = EquirectangularAdapter;
+  export let plugins: (PluginConstructor | [PluginConstructor, unknown])[] = [];
+  export let navbar = false;
+
   let container: HTMLDivElement;
   let viewer: Viewer;
 
   onMount(() => {
     viewer = new Viewer({
+      adapter,
+      plugins,
       container,
       panorama,
-      navbar: false,
+      touchmoveTwoFingers: true,
+      mousewheelCtrlKey: false,
+      navbar,
+      maxFov: 180,
+      fisheye: true,
     });
   });
 

web/src/lib/components/asset-viewer/video-wrapper-viewer.svelte

@@ -0,0 +1,15 @@
+<script lang="ts">
+  import { AssetTypeEnum } from '@immich/sdk';
+  import { ProjectionType } from '$lib/constants';
+  import VideoNativeViewer from '$lib/components/asset-viewer/video-native-viewer.svelte';
+  import PanoramaViewer from '$lib/components/asset-viewer/panorama-viewer.svelte';
+
+  export let assetId: string;
+  export let projectionType: string | null | undefined;
+</script>
+
+{#if projectionType === ProjectionType.EQUIRECTANGULAR}
+  <PanoramaViewer asset={{ id: assetId, type: AssetTypeEnum.Video }} />
+{:else}
+  <VideoNativeViewer {assetId} on:onVideoEnded on:onVideoStarted />
+{/if}