Gaussian Splatting

<script>
  import { Canvas } from '@threlte/core'
  import Scene from './Scene.svelte'
</script>

<div>
  <Canvas>
    <Scene />
  </Canvas>
</div>

<style>
  :global(body) {
    margin: 0;
  }

  div {
    width: 100%;
    height: 100%;
  }
</style>

<script lang="ts">
  import { LumaSplatsThree } from '@lumaai/luma-web'
  import { T, asyncWritable, useCache, useTask, useThrelte } from '@threlte/core'
  import { useSuspense } from '@threlte/extras'
  import { onDestroy } from 'svelte'
  import type { CubeTexture, Scene } from 'three'
  import type { LumaSplatsThreeProps } from './types'

  let {
    source,
    mode = 'object',
    loadingAnimationEnabled = false,
    particleRevealEnabled = false,
    enableThreeShaderIntegration = true,
    children,
    ...rest
  }: LumaSplatsThreeProps = $props()

  const { invalidate, renderer, scene } = useThrelte()

  const { remember } = useCache()
  const suspend = useSuspense()

  const captureCubemap = mode === 'env' || mode === 'object-env'

  const splats = suspend(
    asyncWritable(
      remember(() => {
        return new Promise<[LumaSplatsThree, CubeTexture | undefined]>((resolve) => {
          const splats = new LumaSplatsThree({
            source,
            loadingAnimationEnabled,
            particleRevealEnabled,
            enableThreeShaderIntegration
          })

          splats.onLoad = async () => {
            if (captureCubemap) {
              splats.captureCubemap(renderer).then((cubemap) => {
                resolve([splats, cubemap])
              })
            } else {
              resolve([splats, undefined])
            }
          }
        })
      }, [source])
    )
  )

  let preheat =
    particleRevealEnabled && loadingAnimationEnabled ? 400 : loadingAnimationEnabled ? 100 : 10
  let frame = 0
  const { start, stop } = useTask(
    () => {
      frame++
      if (frame >= preheat) {
        stop()
        frame = 0
      }
    },
    { autoStart: false }
  )

  let previousEnvironment: Scene['environment'] = $state(null)
  let previousBackground: Scene['background'] = $state(null)
  let previousBackgroundBluriness: Scene['backgroundBlurriness'] = $state()

  $effect(() => {
    if ($splats && $splats[1]) {
      previousEnvironment = scene.environment
      previousBackground = scene.background
      previousBackgroundBluriness = scene.backgroundBlurriness
      scene.environment = $splats[1]
      scene.background = $splats[1]
      scene.backgroundBlurriness = 0.5
      invalidate()
    }
  })

  onDestroy(() => {
    if (captureCubemap) {
      scene.environment = previousEnvironment
      scene.background = previousBackground
      scene.backgroundBlurriness = previousBackgroundBluriness
      invalidate()
    }
  })
</script>

{#if (mode === 'object' || mode === 'object-env') && $splats && $splats[0]}
  <T
    dispose={false}
    is={$splats[0]}
    oncreate={() => {
      start()
    }}
    {...rest}
  >
    {@render children?.({ ref: $splats[0] })}
  </T>
{/if}

import type { Snippet } from 'svelte'
import type { LumaSplatsThree, LumaSplatsSource } from '@lumaai/luma-web'

export type LumaSplatsThreeProps = LumaSplatsThree & {
  source: LumaSplatsSource
  mode?: 'object' | 'object-env' | 'env'
  loadingAnimationEnabled?: boolean
  particleRevealEnabled?: boolean
  enableThreeShaderIntegration?: boolean
  children?: Snippet<[any]>
  [key: string]: any
}

<script lang="ts">
  import { useStage, useTask, useThrelte } from '@threlte/core'
  import { WaveformMonitor } from 'svelte-tweakpane-ui'

  const { shouldRender, renderStage } = useThrelte()

  const afterRenderStage = useStage('after-render', {
    after: renderStage
  })

  let log = Array(100).fill(0)

  useTask(
    () => {
      log = update(log)
    },
    {
      autoInvalidate: false,
      stage: afterRenderStage
    }
  )

  function update(log: number[]) {
    log.shift()
    log.push(shouldRender() ? 1 : 0)
    return log
  }
</script>

<WaveformMonitor
  value={log}
  min={-1}
  max={2}
/>

<script lang="ts">
  import { T } from '@threlte/core'
  import { GLTF, OrbitControls } from '@threlte/extras'
  import { Checkbox, Folder, FpsGraph, List, Pane, Slider } from 'svelte-tweakpane-ui'
  import type { MeshStandardMaterial } from 'three'
  import { DEG2RAD } from 'three/src/math/MathUtils.js'
  import LumaSplats from './LumaSplatsThree/LumaSplatsThree.svelte'
  import RenderIndicator from './RenderIndicator.svelte'
  import Splat from './Splat/Splat.svelte'

  // <LumaSplatsThree>
  let showLumaSplats = true
  let lumaSplatsMode: 'object' | 'object-env' | 'env' = 'object-env'

  // <Splat>
  let showSplat = true
  let alphaHash = false
  let alphaTest = 0.06
  let toneMapped = true

  // Car
  let showPorsche = true

  let paneExpanded = false

  let gltfMaterials: Record<string, MeshStandardMaterial> | undefined
  $: if (gltfMaterials) {
    Object.values(gltfMaterials).forEach((material) => {
      material.envMapIntensity = 5
    })
  }
</script>

<LumaSplats
  visible={showLumaSplats}
  source="https://lumalabs.ai/capture/4c15c22e-8655-4423-aeac-b08f017dda22"
  mode={lumaSplatsMode}
/>

<Splat
  visible={showSplat}
  position={[1.08, 2.21, -1.99]}
  rotation={[-32.3 * DEG2RAD, -18.5 * DEG2RAD, -6.4 * DEG2RAD]}
  src="https://huggingface.co/cakewalk/splat-data/resolve/main/nike.splat"
  {alphaHash}
  alphaTest={alphaTest > 0 ? alphaTest : undefined}
  {toneMapped}
/>

<GLTF
  visible={showPorsche}
  position={[-1.48, -0.51, 2.15]}
  rotation.y={57 * DEG2RAD}
  scale={0.7}
  url="/models/splat-example/porsche_959.glb"
  bind:materials={gltfMaterials}
/>

<T.PerspectiveCamera
  makeDefault
  position={[0.22, 2.44, 9.06]}
  oncreate={(ref) => {
    ref.lookAt(0, 0, 0)
  }}
  fov={25}
>
  <OrbitControls />
</T.PerspectiveCamera>

<Pane
  position="fixed"
  title="Gaussian Splatting"
  bind:expanded={paneExpanded}
>
  <Folder
    userExpandable={false}
    expanded
    title="Luma"
  >
    <Checkbox
      bind:value={showLumaSplats}
      label="Show LumaSplats"
    />
    {#if showLumaSplats}
      <List
        bind:value={lumaSplatsMode}
        options={{
          object: 'object',
          'object-env': 'object-env',
          env: 'env'
        }}
      />
    {/if}
  </Folder>

  <Folder
    userExpandable={false}
    expanded
    title="Splat"
  >
    <Checkbox
      bind:value={showSplat}
      label="Show Splats"
    />
    {#if showSplat}
      <Checkbox
        bind:value={alphaHash}
        label="alphaHash"
      />

      <Slider
        bind:value={alphaTest}
        label="alphaTest"
        min={0}
        max={1}
        step={0.01}
      />

      <Checkbox
        bind:value={toneMapped}
        label="toneMapped"
      />
    {/if}
  </Folder>

  <Folder title="Porsche">
    <Checkbox
      bind:value={showPorsche}
      label="Show Porsche"
    />
  </Folder>

  <Folder title="Rendering Activity">
    <RenderIndicator />
    <FpsGraph />
  </Folder>
</Pane>

<script lang="ts">
  import { Splat, SplatLoader } from '@pmndrs/vanilla'
  import { T, useLoader, useTask, useThrelte } from '@threlte/core'
  import type { SplatProps } from './types'

  let {
    src,
    alphaHash = false,
    alphaTest = undefined,
    toneMapped = undefined,
    children,
    ...rest
  }: SplatProps = $props()

  const { renderer, camera } = useThrelte()

  const loader = useLoader(SplatLoader, {
    args: [renderer]
  })

  let framesRendered = 0
  const { start, stop } = useTask(
    () => {
      framesRendered++
      // render for 10 frames
      if (framesRendered >= 10) {
        stop()
        framesRendered = 0
      }
    },
    { autoStart: false }
  )
</script>

{#await loader.load(src) then splat}
  <T
    {...rest}
    dispose={false}
    is={Splat}
    args={[
      splat,
      $camera,
      {
        alphaHash,
        alphaTest,
        toneMapped
      }
    ]}
    oncreate={start}
  >
    {@render children?.({ ref: Splat })}
  </T>
{/await}

import type { Snippet } from 'svelte'

export type SplatProps = {
  src: string
  alphaHash?: boolean
  alphaTest?: number | undefined
  toneMapped?: boolean | undefined
  children?: Snippet<[any]>
  [key: string]: any
}