feat(web): wasm justified layout, sync edition (#23194)

* the invisible wasm

use npm version

* deterministic tests

* add todo

* linting

* bump library, add helpers

* use target height for unfilled rows

* update tests

pnpm-lock.yaml

@@ -677,6 +677,9 @@ importers:
       '@formatjs/icu-messageformat-parser':
         specifier: ^2.9.8
         version: 2.11.4
+      '@immich/justified-layout-wasm':
+        specifier: ^0.4.3
+        version: 0.4.3
       '@immich/sdk':
         specifier: file:../open-api/typescript-sdk
         version: link:../open-api/typescript-sdk
@@ -2726,6 +2729,9 @@ packages:
     cpu: [x64]
     os: [win32]
 
+  '@immich/justified-layout-wasm@0.4.3':
+    resolution: {integrity: sha512-fpcQ7zPhP3Cp1bEXhONVYSUeIANa2uzaQFGKufUZQo5FO7aFT77szTVChhlCy4XaVy5R4ZvgSkA/1TJmeORz7Q==}
+
   '@immich/ui@0.37.1':
     resolution: {integrity: sha512-8S9KsyqyRcNgRHeBU8G3qMQ7D7fN4u9I31jjRc9c3s2tkiYucASofPJdcFdmGZnKLX5fIj+yofxiNZV9tVitOg==}
     peerDependencies:
@@ -14182,6 +14188,8 @@ snapshots:
   '@img/sharp-win32-x64@0.34.4':
     optional: true
 
+  '@immich/justified-layout-wasm@0.4.3': {}
+
   '@immich/ui@0.37.1(@internationalized/date@3.8.2)(svelte@5.40.1)':
     dependencies:
       '@mdi/js': 7.4.47

web/eslint.config.js

@@ -122,6 +122,7 @@ export default typescriptEslint.config(
       'unicorn/prefer-top-level-await': 'off',
       'unicorn/import-style': 'off',
       'unicorn/no-array-sort': 'off',
+      'unicorn/no-for-loop': 'off',
       'svelte/button-has-type': 'error',
       '@typescript-eslint/await-thenable': 'error',
       '@typescript-eslint/no-floating-promises': 'error',

web/package.json

@@ -26,6 +26,7 @@
   },
   "dependencies": {
     "@formatjs/icu-messageformat-parser": "^2.9.8",
+    "@immich/justified-layout-wasm": "^0.4.3",
     "@immich/sdk": "file:../open-api/typescript-sdk",
     "@immich/ui": "^0.37.1",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",

web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte

@@ -16,7 +16,7 @@
   import { archiveAssets, cancelMultiselect } from '$lib/utils/asset-utils';
   import { moveFocus } from '$lib/utils/focus-util';
   import { handleError } from '$lib/utils/handle-error';
-  import { getJustifiedLayoutFromAssets, type CommonJustifiedLayout } from '$lib/utils/layout-utils';
+  import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';
   import { navigate } from '$lib/utils/navigation';
   import { isTimelineAsset, toTimelineAsset } from '$lib/utils/timeline-util';
   import { AssetVisibility, type AssetResponseDto } from '@immich/sdk';
@@ -28,7 +28,7 @@
 
   interface Props {
     initialAssetId?: string;
-    assets: (TimelineAsset | AssetResponseDto)[];
+    assets: TimelineAsset[] | AssetResponseDto[];
     assetInteraction: AssetInteraction;
     disableAssetSelect?: boolean;
     showArchiveIcon?: boolean;
@@ -66,60 +66,26 @@
 
   let { isViewing: isViewerOpen, asset: viewingAsset, setAssetId } = assetViewingStore;
 
-  let geometry: CommonJustifiedLayout | undefined = $state();
+  const geometry = $derived(
-
+    getJustifiedLayoutFromAssets(assets, {
-  $effect(() => {
-    const _assets = assets;
-    updateSlidingWindow();
-
-    const rowWidth = Math.floor(viewport.width);
-    const rowHeight = rowWidth < 850 ? 100 : 235;
-
-    geometry = getJustifiedLayoutFromAssets(_assets, {
       spacing: 2,
-      heightTolerance: 0.15,
+      heightTolerance: 0.5,
-      rowHeight,
+      rowHeight: Math.floor(viewport.width) < 850 ? 100 : 235,
-      rowWidth,
+      rowWidth: Math.floor(viewport.width),
-    });
+    }),
-  });
+  );
 
-  let assetLayouts = $derived.by(() => {
+  const getStyle = (i: number) => {
-    const assetLayout = [];
+    const geo = geometry;
-    let containerHeight = 0;
+    return `top: ${geo.getTop(i)}px; left: ${geo.getLeft(i)}px; width: ${geo.getWidth(i)}px; height: ${geo.getHeight(i)}px;`;
-    let containerWidth = 0;
+  };
-    if (geometry) {
-      containerHeight = geometry.containerHeight;
-      containerWidth = geometry.containerWidth;
-      for (const [index, asset] of assets.entries()) {
-        const top = geometry.getTop(index);
-        const left = geometry.getLeft(index);
-        const width = geometry.getWidth(index);
-        const height = geometry.getHeight(index);
-
-        const layoutTopWithOffset = top + pageHeaderOffset;
-        const layoutBottom = layoutTopWithOffset + height;
-
-        const display = layoutTopWithOffset < slidingWindow.bottom && layoutBottom > slidingWindow.top;
-
-        const layout = {
-          asset,
-          top,
-          left,
-          width,
-          height,
-          display,
-        };
-
-        assetLayout.push(layout);
-      }
-    }
 
-    return {
+  const isIntersecting = (i: number) => {
-      assetLayout,
+    const geo = geometry;
-      containerHeight,
+    const window = slidingWindow;
-      containerWidth,
+    const top = geo.getTop(i);
-    };
+    return top + pageHeaderOffset < window.bottom && top + geo.getHeight(i) > window.top;
-  });
+  };
 
   let currentIndex = 0;
   if (initialAssetId && assets.length > 0) {
@@ -148,9 +114,9 @@
   let lastIntersectedHeight = 0;
   $effect(() => {
     // Intersect if there's only one viewport worth of assets left to scroll.
-    if (assetLayouts.containerHeight - slidingWindow.bottom <= viewport.height) {
+    if (geometry.containerHeight - slidingWindow.bottom <= viewport.height) {
       // Notify we got to (near) the end of scroll.
-      const intersectedHeight = assetLayouts.containerHeight;
+      const intersectedHeight = geometry.containerHeight;
       if (lastIntersectedHeight !== intersectedHeight) {
         debouncedOnIntersected();
         lastIntersectedHeight = intersectedHeight;
@@ -264,7 +230,7 @@
     isShowDeleteConfirmation = false;
     await deleteAssets(
       !(isTrashEnabled && !force),
-      (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id))),
+      (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id)) as TimelineAsset[]),
       assetInteraction.selectedAssets,
       onReload,
     );
@@ -277,7 +243,7 @@
       assetInteraction.isAllArchived ? AssetVisibility.Timeline : AssetVisibility.Archive,
     );
     if (ids) {
-      assets = assets.filter((asset) => !ids.includes(asset.id));
+      assets = assets.filter((asset) => !ids.includes(asset.id)) as TimelineAsset[];
       deselectAllAssets();
     }
   };
@@ -480,41 +446,36 @@
 {#if assets.length > 0}
   <div
     style:position="relative"
-    style:height={assetLayouts.containerHeight + 'px'}
+    style:height={geometry.containerHeight + 'px'}
-    style:width={assetLayouts.containerWidth - 1 + 'px'}
+    style:width={geometry.containerWidth + 'px'}
   >
-    {#each assetLayouts.assetLayout as layout, layoutIndex (layout.asset.id + '-' + layoutIndex)}
+    {#each assets as asset, i (asset.id + '-' + i)}
-      {@const currentAsset = layout.asset}
+      {#if isIntersecting(i)}
-
+        {@const currentAsset = toTimelineAsset(asset)}
-      {#if layout.display}
+        <div class="absolute" style:overflow="clip" style={getStyle(i)}>
-        <div
-          class="absolute"
-          style:overflow="clip"
-          style="width: {layout.width}px; height: {layout.height}px; top: {layout.top}px; left: {layout.left}px"
-        >
           <Thumbnail
             readonly={disableAssetSelect}
             onClick={() => {
               if (assetInteraction.selectionActive) {
-                handleSelectAssets(toTimelineAsset(currentAsset));
+                handleSelectAssets(currentAsset);
                 return;
               }
-              void viewAssetHandler(toTimelineAsset(currentAsset));
+              void viewAssetHandler(currentAsset);
             }}
-            onSelect={() => handleSelectAssets(toTimelineAsset(currentAsset))}
+            onSelect={() => handleSelectAssets(currentAsset)}
-            onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(currentAsset))}
+            onMouseEvent={() => assetMouseEventHandler(currentAsset)}
             {showArchiveIcon}
-            asset={toTimelineAsset(currentAsset)}
+            asset={currentAsset}
             selected={assetInteraction.hasSelectedAsset(currentAsset.id)}
             selectionCandidate={assetInteraction.hasSelectionCandidate(currentAsset.id)}
-            thumbnailWidth={layout.width}
+            thumbnailWidth={geometry.getWidth(i)}
-            thumbnailHeight={layout.height}
+            thumbnailHeight={geometry.getHeight(i)}
           />
-          {#if showAssetName && !isTimelineAsset(currentAsset)}
+          {#if showAssetName && !isTimelineAsset(asset)}
             <div
               class="absolute text-center p-1 text-xs font-mono font-semibold w-full bottom-0 bg-linear-to-t bg-slate-50/75 dark:bg-slate-800/75 overflow-clip text-ellipsis whitespace-pre-wrap"
             >
-              {currentAsset.originalFileName}
+              {asset.originalFileName}
             </div>
           {/if}
         </div>

web/src/lib/managers/VirtualScrollManager/VirtualScrollManager.svelte.ts

@@ -26,12 +26,12 @@ export abstract class VirtualScrollManager {
   #suspendTransitions = $state(false);
   #resetScrolling = debounce(() => (this.#scrolling = false), 1000);
   #resetSuspendTransitions = debounce(() => (this.suspendTransitions = false), 1000);
-  #justifiedLayoutOptions = $derived.by(() => ({
+  #justifiedLayoutOptions = $derived({
     spacing: 2,
-    heightTolerance: 0.15,
+    heightTolerance: 0.5,
     rowHeight: this.#rowHeight,
     rowWidth: Math.floor(this.viewportWidth),
-  }));
+  });
 
   constructor() {
     this.setLayoutOptions();

web/src/lib/managers/timeline-manager/day-group.svelte.ts

@@ -1,7 +1,7 @@
 import { AssetOrder } from '@immich/sdk';
 
 import type { CommonLayoutOptions } from '$lib/utils/layout-utils';
-import { getJustifiedLayoutFromAssets, getPosition } from '$lib/utils/layout-utils';
+import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';
 import { plainDateTimeCompare } from '$lib/utils/timeline-util';
 
 import { SvelteSet } from 'svelte/reactivity';
@@ -148,9 +148,9 @@ export class DayGroup {
     const geometry = getJustifiedLayoutFromAssets(assets, options);
     this.width = geometry.containerWidth;
     this.height = assets.length === 0 ? 0 : geometry.containerHeight;
+    // TODO: lazily get positions instead of loading them all here
     for (let i = 0; i < this.viewerAssets.length; i++) {
-      const position = getPosition(geometry, i);
+      this.viewerAssets[i].position = geometry.getPosition(i);
-      this.viewerAssets[i].position = position;
     }
   }
 

web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts

@@ -82,15 +82,15 @@ describe('TimelineManager', () => {
 
       expect(plainMonths).toEqual(
         expect.arrayContaining([
-          expect.objectContaining({ year: 2024, month: 3, height: 165.5 }),
+          expect.objectContaining({ year: 2024, month: 3, height: 283 }),
-          expect.objectContaining({ year: 2024, month: 2, height: 11_996 }),
+          expect.objectContaining({ year: 2024, month: 2, height: 7711 }),
           expect.objectContaining({ year: 2024, month: 1, height: 286 }),
         ]),
       );
     });
 
     it('calculates timeline height', () => {
-      expect(timelineManager.totalViewerHeight).toBe(12_507.5);
+      expect(timelineManager.totalViewerHeight).toBe(8340);
     });
   });
 

web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts

@@ -18,7 +18,7 @@ export class ViewerAsset {
     return calculateViewerAssetIntersecting(store, positionTop, this.position.height);
   });
 
-  position: CommonPosition | undefined = $state();
+  position: CommonPosition | undefined = $state.raw();
   asset: TimelineAsset = <TimelineAsset>$state();
   id: string = $derived(this.asset.id);
 

web/src/lib/utils/layout-utils.ts

@@ -1,16 +1,15 @@
-// import { TUNABLES } from '$lib/utils/tunables';
+import { TUNABLES } from '$lib/utils/tunables';
-// note: it's important that this is not imported in more than one file due to https://github.com/sveltejs/kit/issues/7805
+import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
-// import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
 
 import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
 import { getAssetRatio } from '$lib/utils/asset-utils';
-import { isTimelineAsset } from '$lib/utils/timeline-util';
+import { isTimelineAsset, isTimelineAssets } from '$lib/utils/timeline-util';
 import type { AssetResponseDto } from '@immich/sdk';
 import createJustifiedLayout from 'justified-layout';
 
 export type getJustifiedLayoutFromAssetsFunction = typeof getJustifiedLayoutFromAssets;
 
-// let useWasm = TUNABLES.LAYOUT.WASM;
+const useWasm = TUNABLES.LAYOUT.WASM;
 
 export type CommonJustifiedLayout = {
   containerWidth: number;
@@ -19,6 +18,7 @@ export type CommonJustifiedLayout = {
   getLeft(boxIdx: number): number;
   getWidth(boxIdx: number): number;
   getHeight(boxIdx: number): number;
+  getPosition(boxIdx: number): { top: number; left: number; width: number; height: number };
 };
 
 export type CommonLayoutOptions = {
@@ -29,25 +29,31 @@ export type CommonLayoutOptions = {
 };
 
 export function getJustifiedLayoutFromAssets(
-  assets: (TimelineAsset | AssetResponseDto)[],
+  assets: TimelineAsset[] | AssetResponseDto[],
   options: CommonLayoutOptions,
 ): CommonJustifiedLayout {
-  // if (useWasm) {
+  if (useWasm) {
-  //   return wasmJustifiedLayout(assets, options);
+    return isTimelineAssets(assets) ? wasmLayoutFromTimeline(assets, options) : wasmLayoutFromDto(assets, options);
-  // }
+  }
   return justifiedLayout(assets, options);
 }
 
-// commented out until a solution for top level awaits on safari is fixed
+function wasmLayoutFromTimeline(assets: TimelineAsset[], options: LayoutOptions) {
-// function wasmJustifiedLayout(assets: AssetResponseDto[], options: LayoutOptions) {
+  const aspectRatios = new Float32Array(assets.length);
-//   const aspectRatios = new Float32Array(assets.length);
+  for (let i = 0; i < assets.length; i++) {
-//   // eslint-disable-next-line unicorn/no-for-loop
+    aspectRatios[i] = assets[i].ratio;
-//   for (let i = 0; i < assets.length; i++) {
+  }
-//     const { width, height } = getAssetRatio(assets[i]);
+  return new JustifiedLayout(aspectRatios, options);
-//     aspectRatios[i] = width / height;
+}
-//   }
+
-//   return new JustifiedLayout(aspectRatios, options);
+function wasmLayoutFromDto(assets: AssetResponseDto[], options: LayoutOptions) {
-// }
+  const aspectRatios = new Float32Array(assets.length);
+  for (let i = 0; i < assets.length; i++) {
+    const { width, height } = getAssetRatio(assets[i]);
+    aspectRatios[i] = width / height;
+  }
+  return new JustifiedLayout(aspectRatios, options);
+}
 
 type Geometry = ReturnType<typeof createJustifiedLayout>;
 class Adapter {
@@ -88,6 +94,11 @@ class Adapter {
   getHeight(boxIdx: number) {
     return this.result.boxes[boxIdx]?.height;
   }
+
+  getPosition(boxIdx: number) {
+    const box = this.result.boxes[boxIdx];
+    return { top: box.top, left: box.left, width: box.width, height: box.height };
+  }
 }
 
 export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], options: CommonLayoutOptions) {
@@ -119,12 +130,3 @@ export type CommonPosition = {
   width: number;
   height: number;
 };
-
-export function getPosition(geometry: CommonJustifiedLayout, boxIdx: number): CommonPosition {
-  const top = geometry.getTop(boxIdx);
-  const left = geometry.getLeft(boxIdx);
-  const width = geometry.getWidth(boxIdx);
-  const height = geometry.getHeight(boxIdx);
-
-  return { top, left, width, height };
-}

web/src/lib/utils/timeline-util.ts

@@ -195,6 +195,9 @@ export const toTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset):
 export const isTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset): unknownAsset is TimelineAsset =>
   (unknownAsset as TimelineAsset).ratio !== undefined;
 
+export const isTimelineAssets = (assets: AssetResponseDto[] | TimelineAsset[]): assets is TimelineAsset[] =>
+  assets.length === 0 || 'ratio' in assets[0];
+
 export const plainDateTimeCompare = (ascending: boolean, a: TimelineDateTime, b: TimelineDateTime) => {
   const [aDateTime, bDateTime] = ascending ? [a, b] : [b, a];
 

web/src/lib/utils/tunables.ts

@@ -19,7 +19,7 @@ const storage = browser
     };
 export const TUNABLES = {
   LAYOUT: {
-    WASM: getBoolean(storage.getItem('LAYOUT.WASM'), false),
+    WASM: getBoolean(storage.getItem('LAYOUT.WASM'), true),
   },
   TIMELINE: {
     INTERSECTION_EXPAND_TOP: getNumber(storage.getItem('TIMELINE_INTERSECTION_EXPAND_TOP'), 500),

web/src/test-data/factories/asset-factory.ts

@@ -32,7 +32,7 @@ export const assetFactory = Sync.makeFactory<AssetResponseDto>({
 
 export const timelineAssetFactory = Sync.makeFactory<TimelineAsset>({
   id: Sync.each(() => faker.string.uuid()),
-  ratio: Sync.each(() => faker.number.int()),
+  ratio: Sync.each((i) => 0.2 + ((i * 0.618_034) % 3.8)), // deterministic random float between 0.2 and 4.0
   ownerId: Sync.each(() => faker.string.uuid()),
   thumbhash: Sync.each(() => faker.string.alphanumeric(28)),
   localDateTime: Sync.each(() => fromISODateTimeUTCToObject(faker.date.past().toISOString())),