├── .nvmrc
├── .gitignore
├── packages
├── app
│ ├── .env
│ ├── .eslintignore
│ ├── src
│ │ ├── react-app-env.d.ts
│ │ ├── polyfills.ts
│ │ ├── global.d.ts
│ │ ├── webxr-polyfill.d.ts
│ │ ├── components
│ │ │ ├── ui
│ │ │ │ ├── GroupControl.tsx
│ │ │ │ ├── Control.tsx
│ │ │ │ ├── GroupControlElement.tsx
│ │ │ │ └── UI.tsx
│ │ │ ├── util
│ │ │ │ └── ConditionalWrapper.tsx
│ │ │ ├── VrPlayer.tsx
│ │ │ ├── DebugPlayer.tsx
│ │ │ └── App.tsx
│ │ ├── setupTests.ts
│ │ ├── util
│ │ │ └── debounce.ts
│ │ ├── atoms
│ │ │ └── controls.ts
│ │ ├── worker
│ │ │ └── videoRecognition.worker
│ │ │ │ ├── index.ts
│ │ │ │ ├── videoRecognition.spec.ts
│ │ │ │ └── videoRecognition.ts
│ │ ├── index.css
│ │ ├── helper
│ │ │ ├── util.ts
│ │ │ └── getImageFrames.ts
│ │ ├── index.tsx
│ │ ├── hooks
│ │ │ └── useXRSession.ts
│ │ ├── logo.svg
│ │ ├── service-worker.ts
│ │ └── serviceWorkerRegistration.ts
│ ├── public
│ │ ├── robots.txt
│ │ ├── favicon.ico
│ │ ├── thumbnail.png
│ │ ├── favicon-16x16.png
│ │ ├── favicon-32x32.png
│ │ ├── mstile-150x150.png
│ │ ├── apple-touch-icon.png
│ │ ├── android-chrome-192x192.png
│ │ ├── android-chrome-512x512.png
│ │ ├── browserconfig.xml
│ │ ├── manifest.json
│ │ ├── index.html
│ │ └── safari-pinned-tab.svg
│ ├── postcss.config.js
│ ├── tailwind.config.js
│ ├── .gitignore
│ ├── .eslintrc
│ ├── tsconfig.json
│ ├── package.json
│ └── README.md
└── player
│ ├── .eslintignore
│ ├── .gitignore
│ ├── src
│ ├── primitive
│ │ ├── index.ts
│ │ ├── primitive.ts
│ │ ├── square.ts
│ │ └── sphere.ts
│ ├── types.ts
│ ├── index.ts
│ └── renderer
│ │ ├── renderProps.ts
│ │ ├── debugRenderer.ts
│ │ ├── vrRenderer.ts
│ │ └── renderer.ts
│ ├── .eslintrc
│ ├── package.json
│ └── tsconfig.json
├── .npmrc
├── .prettierrc
├── .ncurc.js
├── README.md
├── .editorconfig
├── .vscode
├── settings.json
└── launch.json
├── package.json
├── LICENCE
├── .eslintrc
├── .gitattributes
└── org
└── icon.svg
/.nvmrc:
--------------------------------------------------------------------------------
1 | 16.13
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 |
--------------------------------------------------------------------------------
/packages/app/.env:
--------------------------------------------------------------------------------
1 | BROWSER = 'none'
2 |
--------------------------------------------------------------------------------
/packages/player/.eslintignore:
--------------------------------------------------------------------------------
1 | lib
2 |
--------------------------------------------------------------------------------
/packages/player/.gitignore:
--------------------------------------------------------------------------------
1 | lib
2 |
--------------------------------------------------------------------------------
/.npmrc:
--------------------------------------------------------------------------------
1 | engine-strict=true
2 | legacy-peer-deps=true
3 |
--------------------------------------------------------------------------------
/packages/app/.eslintignore:
--------------------------------------------------------------------------------
1 | build
2 | tailwind.config.js
3 | postcss.config.js
4 |
--------------------------------------------------------------------------------
/packages/app/src/react-app-env.d.ts:
--------------------------------------------------------------------------------
1 | ///
2 |
--------------------------------------------------------------------------------
/packages/player/src/primitive/index.ts:
--------------------------------------------------------------------------------
1 | export * from './primitive';
2 | export * from './sphere';
3 |
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": true,
3 | "trailingComma": "all",
4 | "proseWrap": "always"
5 | }
6 |
--------------------------------------------------------------------------------
/packages/app/public/robots.txt:
--------------------------------------------------------------------------------
1 | # https://www.robotstxt.org/robotstxt.html
2 | User-agent: *
3 | Disallow:
4 |
--------------------------------------------------------------------------------
/packages/app/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/favicon.ico
--------------------------------------------------------------------------------
/packages/app/public/thumbnail.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/thumbnail.png
--------------------------------------------------------------------------------
/packages/app/public/favicon-16x16.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/favicon-16x16.png
--------------------------------------------------------------------------------
/packages/app/public/favicon-32x32.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/favicon-32x32.png
--------------------------------------------------------------------------------
/packages/app/public/mstile-150x150.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/mstile-150x150.png
--------------------------------------------------------------------------------
/packages/app/postcss.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/packages/app/public/apple-touch-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/apple-touch-icon.png
--------------------------------------------------------------------------------
/packages/app/src/polyfills.ts:
--------------------------------------------------------------------------------
1 | import WebXRPolyfill from 'webxr-polyfill';
2 |
3 | // eslint-disable-next-line no-new
4 | new WebXRPolyfill();
5 |
--------------------------------------------------------------------------------
/packages/app/public/android-chrome-192x192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/android-chrome-192x192.png
--------------------------------------------------------------------------------
/packages/app/public/android-chrome-512x512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/TimoWilhelm/vr-player/HEAD/packages/app/public/android-chrome-512x512.png
--------------------------------------------------------------------------------
/packages/app/src/global.d.ts:
--------------------------------------------------------------------------------
1 | declare global {
2 | interface Navigator extends Navigator {
3 | xr: XRSystem;
4 | }
5 | }
6 |
7 | export {};
8 |
--------------------------------------------------------------------------------
/packages/player/src/types.ts:
--------------------------------------------------------------------------------
1 | export type Format = 'screen' | '180' | '360';
2 | export type Layout = 'mono' | 'stereoTopBottom' | 'stereoLeftRight';
3 |
--------------------------------------------------------------------------------
/packages/app/src/webxr-polyfill.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'webxr-polyfill' {
2 | declare class WebXRPolyfill {
3 | xr: XRSystem;
4 | }
5 | export = WebXRPolyfill;
6 | }
7 |
8 | export {};
9 |
--------------------------------------------------------------------------------
/packages/player/src/index.ts:
--------------------------------------------------------------------------------
1 | export { VrRenderer } from './renderer/vrRenderer';
2 | export { DebugRenderer } from './renderer/debugRenderer';
3 | export type { Format, Layout } from './types';
4 |
--------------------------------------------------------------------------------
/.ncurc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | format: 'group',
3 | workspaces: true,
4 | root: true,
5 | upgrade: true,
6 | reject: [
7 | // breaking
8 | 'typescript',
9 | 'eslint',
10 | ],
11 | };
12 |
--------------------------------------------------------------------------------
/packages/player/src/primitive/primitive.ts:
--------------------------------------------------------------------------------
1 | import type { vec2, vec3 } from 'gl-matrix';
2 |
3 | export type Primitive = {
4 | positions: vec3[];
5 | indices: vec3[];
6 | uvs: vec2[];
7 | normals: vec3[];
8 | };
9 |
--------------------------------------------------------------------------------
/packages/player/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "extends": [
3 | "../../.eslintrc"
4 | ],
5 | "parserOptions": {
6 | "sourceType": "module",
7 | "project": [
8 | "./tsconfig.json"
9 | ]
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/packages/app/tailwind.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | content: ['./src/**/*.{js,jsx,ts,tsx}'],
3 | theme: {
4 | extend: {
5 | aria: {
6 | current: 'current="true"',
7 | },
8 | },
9 | },
10 | plugins: [],
11 | };
12 |
--------------------------------------------------------------------------------
/packages/app/src/components/ui/GroupControl.tsx:
--------------------------------------------------------------------------------
1 | export function GroupControl({ children }: { children: React.ReactNode }) {
2 | return (
3 |
4 | {children}
5 |
6 | );
7 | }
8 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # VR Player
2 |
3 | Play any 180° or 360° videos on your VR HMD using the WebXR APIs. Most common VR headsets should be supported.
4 |
5 | Supports mono or stereo video playback with top-bottom or left-right layouts.
6 |
7 | ---
8 |
9 | Icons by [unDraw](https://undraw.co/)
10 |
--------------------------------------------------------------------------------
/packages/app/src/setupTests.ts:
--------------------------------------------------------------------------------
1 | // jest-dom adds custom jest matchers for asserting on DOM nodes.
2 | // allows you to do things like:
3 | // expect(element).toHaveTextContent(/react/i)
4 | // learn more: https://github.com/testing-library/jest-dom
5 | import '@testing-library/jest-dom';
6 | import 'jest-canvas-mock';
7 |
--------------------------------------------------------------------------------
/packages/app/public/browserconfig.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | #da532c
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/packages/app/src/components/util/ConditionalWrapper.tsx:
--------------------------------------------------------------------------------
1 | export const ConditionalWrapper = ({
2 | condition,
3 | wrapper,
4 | children,
5 | }: {
6 | condition: boolean;
7 | wrapper: (children: React.ReactElement) => JSX.Element;
8 | children: React.ReactElement;
9 | }) => (condition ? wrapper(children) : children);
10 |
--------------------------------------------------------------------------------
/packages/app/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.js
7 |
8 | # testing
9 | /coverage
10 |
11 | # production
12 | /build
13 |
14 | # misc
15 | .DS_Store
16 | .env.local
17 | .env.development.local
18 | .env.test.local
19 | .env.production.local
20 |
21 | npm-debug.log*
22 | yarn-debug.log*
23 | yarn-error.log*
24 |
--------------------------------------------------------------------------------
/packages/player/src/renderer/renderProps.ts:
--------------------------------------------------------------------------------
1 | import type { Texture2D } from 'regl';
2 | import type { mat4 } from 'gl-matrix';
3 |
4 | export type RenderProps = {
5 | model: mat4;
6 | view: mat4 | Float32Array;
7 | projection: mat4 | Float32Array;
8 | texture: Texture2D;
9 | texCoordScaleOffset: Float32Array;
10 | viewport: {
11 | x: GLint;
12 | y: GLint;
13 | width: GLsizei;
14 | height: GLsizei;
15 | };
16 | };
17 |
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | # EditorConfig is awesome: http://EditorConfig.org
2 |
3 | root = true
4 |
5 | [*]
6 | charset = utf-8
7 | end_of_line = lf
8 | indent_style = space
9 | indent_size = 2
10 | insert_final_newline = true
11 | trim_trailing_whitespace = true
12 |
13 | [*.{md,mdx}]
14 | max_line_length = 120
15 | trim_trailing_whitespace = false
16 |
17 | [*.{ts,tsx,js,jsx,mjs}]
18 | quote_type = single
19 |
20 | [*.{yaml,yml}]
21 | quote_type = single
22 |
--------------------------------------------------------------------------------
/packages/app/src/util/debounce.ts:
--------------------------------------------------------------------------------
1 | // eslint-disable-next-line @typescript-eslint/no-explicit-any
2 | export function debounceAnimationFrame void>(
3 | fn: T,
4 | ): T {
5 | let handle: number | undefined;
6 |
7 | return ((...args: unknown[]) => {
8 | if (handle !== undefined) {
9 | window.cancelAnimationFrame(handle);
10 | }
11 |
12 | handle = window.requestAnimationFrame(() => {
13 | fn(...args);
14 | });
15 | }) as T;
16 | }
17 |
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "files.associations": {
3 | "*.css": "tailwindcss"
4 | },
5 | "editor.quickSuggestions": {
6 | "strings": true
7 | },
8 | "debug.internalConsoleOptions": "neverOpen",
9 | "eslint.validate": [
10 | "javascript",
11 | "javascriptreact",
12 | "typescript",
13 | "typescriptreact",
14 | ],
15 | "eslint.workingDirectories": [
16 | {
17 | "mode": "auto"
18 | }
19 | ],
20 | "editor.codeActionsOnSave": {
21 | "source.fixAll": "explicit"
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/.vscode/launch.json:
--------------------------------------------------------------------------------
1 | {
2 | // Use IntelliSense to learn about possible attributes.
3 | // Hover to view descriptions of existing attributes.
4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
5 | "version": "0.2.0",
6 | "configurations": [
7 | {
8 | "name": "App:Chrome",
9 | "type": "chrome",
10 | "request": "launch",
11 | "url": "http://localhost:3000",
12 | "webRoot": "${workspaceFolder}/packages/app",
13 | "sourceMaps": true
14 | },
15 | ]
16 | }
17 |
--------------------------------------------------------------------------------
/packages/app/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "env": {
3 | "jest": true,
4 | },
5 | "plugins": ["react"],
6 | "extends": [
7 | "plugin:react/recommended",
8 | "plugin:react/jsx-runtime",
9 | "../../.eslintrc",
10 | ],
11 | "parserOptions": {
12 | "sourceType": "module",
13 | "project": ["./tsconfig.json"],
14 | },
15 | "settings": {
16 | "react": {
17 | "version": "detect",
18 | },
19 | "import/resolver": {
20 | "node": {
21 | "paths": ["src"],
22 | },
23 | },
24 | },
25 | }
26 |
--------------------------------------------------------------------------------
/packages/app/src/atoms/controls.ts:
--------------------------------------------------------------------------------
1 | import { atom } from 'jotai';
2 | import type { Format, Layout } from '@vr-viewer/player';
3 |
4 | export const autoPlayAtom = atom(false);
5 | export const autoDetectAtom = atom(true);
6 | export const detectingAtom = atom(false);
7 |
8 | export const layoutAtom = atom('mono');
9 | export const flipLayoutAtom = atom(false);
10 |
11 | export const formatAtom = atom('screen');
12 |
13 | export const debugAtom = atom(false);
14 |
15 | export const videoUrlAtom = atom(null);
16 |
--------------------------------------------------------------------------------
/packages/app/src/worker/videoRecognition.worker/index.ts:
--------------------------------------------------------------------------------
1 | import { detectFormat, detectLayout } from './videoRecognition';
2 | import { expose } from 'comlink';
3 | import type { Format, Layout } from '@vr-viewer/player';
4 |
5 | export function recognizeVideo(frames: ImageData[]): [Layout?, Format?] {
6 | const layout = detectLayout(frames);
7 | const format = detectFormat(frames);
8 |
9 | return [layout, format];
10 | }
11 |
12 | const module = { recognizeVideo };
13 |
14 | expose(module);
15 |
16 | export type VideoRecognitionWorker = typeof module;
17 |
--------------------------------------------------------------------------------
/packages/app/src/components/ui/Control.tsx:
--------------------------------------------------------------------------------
1 | import clsx from 'clsx';
2 |
3 | export function Control(
4 | props: Omit, 'type' | 'className'>,
5 | ) {
6 | return (
7 |
14 | );
15 | }
16 |
--------------------------------------------------------------------------------
/packages/player/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@vr-viewer/player",
3 | "version": "0.1.0",
4 | "private": true,
5 | "main": "lib/index.js",
6 | "types": "lib/index.d.ts",
7 | "dependencies": {
8 | "@types/webxr": "^0.5.20",
9 | "gl-matrix": "^3.4.3",
10 | "regl": "^2.1.0",
11 | "typescript": "^5.4.3"
12 | },
13 | "scripts": {
14 | "watch": "tsc --watch",
15 | "build": "tsc",
16 | "test": "echo \"Error: no test specified\" && exit 1",
17 | "lint": "eslint .",
18 | "update": "npx npm-check-updates -u"
19 | },
20 | "author": "",
21 | "license": "MIT"
22 | }
23 |
--------------------------------------------------------------------------------
/packages/app/src/components/ui/GroupControlElement.tsx:
--------------------------------------------------------------------------------
1 | import clsx from 'clsx';
2 |
3 | export function GroupControlElement(
4 | props: Omit, 'type' | 'className'>,
5 | ) {
6 | return (
7 |
14 | );
15 | }
16 |
--------------------------------------------------------------------------------
/packages/app/src/index.css:
--------------------------------------------------------------------------------
1 | @tailwind base;
2 | @tailwind components;
3 | @tailwind utilities;
4 |
5 | @layer base {
6 | html,
7 | body,
8 | #root {
9 | height: 100%;
10 | @apply bg-gray-900;
11 | }
12 | }
13 |
14 | @layer components {
15 | .grid-effect {
16 | background: linear-gradient(
17 | 180deg,
18 | transparent 0,
19 | theme('colors.slate.800') 66vh
20 | ),
21 | fixed 0 0 /20px 20px radial-gradient(rgba(236, 236, 236, 0.2) 1px, transparent
22 | 0),
23 | fixed 10px 10px /20px 20px radial-gradient(rgba(236, 236, 236, 0.2) 1px, transparent
24 | 0);
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/packages/player/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "es5",
4 | "lib": [
5 | "ESNext",
6 | "DOM",
7 | ],
8 | "allowJs": false,
9 | "skipLibCheck": true,
10 | "esModuleInterop": true,
11 | "allowSyntheticDefaultImports": true,
12 | "strict": true,
13 | "forceConsistentCasingInFileNames": true,
14 | "noFallthroughCasesInSwitch": true,
15 | "module": "esnext",
16 | "moduleResolution": "node",
17 | "resolveJsonModule": true,
18 | "isolatedModules": true,
19 | "declaration": true,
20 | "sourceMap": true,
21 | "outDir": "./lib",
22 | },
23 | "include": [
24 | "src"
25 | ]
26 | }
27 |
--------------------------------------------------------------------------------
/packages/player/src/primitive/square.ts:
--------------------------------------------------------------------------------
1 | import { vec3 } from 'gl-matrix';
2 | import type { Primitive } from './primitive';
3 |
4 | export function square({ scale = 1 }): Primitive {
5 | return {
6 | positions: [
7 | vec3.scale(vec3.create(), [-1, 1, 0], scale),
8 | vec3.scale(vec3.create(), [-1, -1, 0], scale),
9 | vec3.scale(vec3.create(), [1, -1, 0], scale),
10 | vec3.scale(vec3.create(), [1, 1, 0], scale),
11 | ],
12 | indices: [
13 | [0, 1, 2],
14 | [0, 2, 3],
15 | ],
16 | uvs: [
17 | [1, 1],
18 | [1, 0],
19 | [0, 0],
20 | [0, 1],
21 | ],
22 | normals: [
23 | [0, 0, -1],
24 | [0, 0, -1],
25 | ],
26 | };
27 | }
28 |
--------------------------------------------------------------------------------
/packages/app/public/manifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "short_name": "VR Player",
3 | "name": "VR Player",
4 | "icons": [
5 | {
6 | "src": "/android-chrome-192x192.png",
7 | "sizes": "192x192",
8 | "type": "image/png",
9 | "purpose": "any"
10 | },
11 | {
12 | "src": "/android-chrome-512x512.png",
13 | "sizes": "512x512",
14 | "type": "image/png",
15 | "purpose": "any"
16 | },
17 | {
18 | "src": "/apple-touch-icon.png",
19 | "sizes": "180x180",
20 | "type": "image/png",
21 | "purpose": "maskable"
22 | }
23 | ],
24 | "start_url": ".",
25 | "display": "standalone",
26 | "theme_color": "#ffffff",
27 | "background_color": "#ffffff"
28 | }
29 |
--------------------------------------------------------------------------------
/packages/app/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "baseUrl": "src",
4 | "target": "es5",
5 | "lib": [
6 | "ESNext",
7 | "DOM",
8 | "DOM.Iterable"
9 | ],
10 | "allowJs": false,
11 | "skipLibCheck": true,
12 | "esModuleInterop": true,
13 | "allowSyntheticDefaultImports": true,
14 | "strict": true,
15 | "forceConsistentCasingInFileNames": true,
16 | "noFallthroughCasesInSwitch": true,
17 | "module": "esnext",
18 | "moduleResolution": "node",
19 | "resolveJsonModule": true,
20 | "isolatedModules": true,
21 | "noEmit": true,
22 | "jsx": "react-jsx",
23 | "experimentalDecorators": true
24 | },
25 | "include": [
26 | "src"
27 | ]
28 | }
29 |
--------------------------------------------------------------------------------
/packages/app/src/helper/util.ts:
--------------------------------------------------------------------------------
1 | export function* nWise(
2 | n: number,
3 | iterable: Iterable,
4 | ): Generator> {
5 | const iterator = iterable[Symbol.iterator]();
6 | let current = iterator.next();
7 |
8 | let tmp = [];
9 |
10 | while (!current.done) {
11 | tmp.push(current.value);
12 | if (tmp.length === n) {
13 | yield tmp;
14 | tmp = [];
15 | }
16 |
17 | current = iterator.next();
18 | }
19 | }
20 |
21 | export function linSpace(
22 | startValue: number,
23 | stopValue: number,
24 | cardinality: number,
25 | ) {
26 | const arr = [];
27 | const step = (stopValue - startValue) / (cardinality - 1);
28 | for (let i = 0; i < cardinality; i += 1) {
29 | arr.push(startValue + step * i);
30 | }
31 | return arr;
32 | }
33 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "workspaces": [
3 | "./packages/player",
4 | "./packages/app"
5 | ],
6 | "devDependencies": {
7 | "@typescript-eslint/eslint-plugin": "^8.8.0",
8 | "@typescript-eslint/parser": "^8.8.0",
9 | "eslint": "^8.57.0",
10 | "eslint-config-prettier": "^9.1.0",
11 | "eslint-import-resolver-typescript": "^3.6.3",
12 | "eslint-plugin-import": "^2.30.0",
13 | "eslint-plugin-prettier": "^5.2.1",
14 | "eslint-plugin-sort-imports-es6-autofix": "^0.6.0",
15 | "npm-check-updates": "^17.1.3",
16 | "prettier": "^3.3.3"
17 | },
18 | "scripts": {
19 | "start": "npm run start --workspace packages/app",
20 | "build": "npm run build --workspaces --if-present",
21 | "lint": "npm run lint --workspaces --if-present",
22 | "test": "npm run test --workspaces --if-present",
23 | "update": "ncu && npm install"
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/packages/app/src/components/VrPlayer.tsx:
--------------------------------------------------------------------------------
1 | import { VrRenderer } from '@vr-viewer/player';
2 | import { useEffect } from 'react';
3 | import type { Format, Layout } from '@vr-viewer/player';
4 |
5 | export function VrPlayer({
6 | xrSession,
7 | video,
8 | canvas,
9 | layout,
10 | flipLayout,
11 | format,
12 | }: {
13 | xrSession: XRSession;
14 | video: HTMLVideoElement;
15 | canvas: HTMLCanvasElement;
16 | layout: Layout;
17 | flipLayout: boolean;
18 | format: Format;
19 | }) {
20 | useEffect(() => {
21 | const renderer = new VrRenderer(
22 | xrSession,
23 | video,
24 | canvas,
25 | layout,
26 | flipLayout,
27 | format,
28 | );
29 | void renderer.start();
30 | return () => {
31 | renderer.stop();
32 | };
33 | }, [canvas, flipLayout, format, layout, video, xrSession]);
34 |
35 | return null;
36 | }
37 |
--------------------------------------------------------------------------------
/packages/app/src/components/DebugPlayer.tsx:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-param-reassign */
2 | import { DebugRenderer } from '@vr-viewer/player';
3 | import { useEffect } from 'react';
4 | import type { Format, Layout } from '@vr-viewer/player';
5 |
6 | export function DebugPlayer({
7 | video,
8 | canvas,
9 | layout,
10 | flipLayout,
11 | format,
12 | view = 'left',
13 | }: {
14 | video: HTMLVideoElement;
15 | canvas: HTMLCanvasElement;
16 | layout: Layout;
17 | flipLayout: boolean;
18 | format: Format;
19 | view?: 'left' | 'right';
20 | }) {
21 | useEffect(() => {
22 | const renderer = new DebugRenderer(
23 | video,
24 | canvas,
25 | layout,
26 | flipLayout,
27 | format,
28 | view,
29 | );
30 | void renderer.start();
31 | return () => {
32 | renderer.stop();
33 | };
34 | }, [canvas, flipLayout, format, layout, video, view]);
35 |
36 | return null;
37 | }
38 |
--------------------------------------------------------------------------------
/LICENCE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 Timo Wilhelm,
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
6 | this software and associated documentation files (the "Software"), to deal in
7 | the Software without restriction, including without limitation the rights to
8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/packages/app/src/index.tsx:
--------------------------------------------------------------------------------
1 | import './index.css';
2 | import './polyfills';
3 | import * as serviceWorkerRegistration from './serviceWorkerRegistration';
4 | import { Analytics } from '@vercel/analytics/react';
5 | import { App } from 'components/App';
6 | import { ConditionalWrapper } from 'components/util/ConditionalWrapper';
7 | import { createRoot } from 'react-dom/client';
8 | import { useAtomsDevtools } from 'jotai-devtools';
9 | import React from 'react';
10 |
11 | const isProduction = process.env.NODE_ENV === 'production';
12 |
13 | const AtomsDevtools = ({ children }: { children: React.ReactElement }) => {
14 | useAtomsDevtools('demo');
15 | return children;
16 | };
17 |
18 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
19 | const root = createRoot(document.getElementById('root')!);
20 |
21 | root.render(
22 |
23 |
24 | {children}}
27 | >
28 |
29 |
30 | ,
31 | );
32 |
33 | serviceWorkerRegistration.register({
34 | onUpdate: (e) => {
35 | if (e.waiting) {
36 | e.waiting.postMessage({ type: 'SKIP_WAITING' });
37 | }
38 |
39 | void e.update().then(() => {
40 | window.location.reload();
41 | });
42 | },
43 | });
44 |
--------------------------------------------------------------------------------
/packages/app/src/helper/getImageFrames.ts:
--------------------------------------------------------------------------------
1 | import { linSpace } from './util';
2 |
3 | export async function getImageFrames(
4 | video: HTMLVideoElement,
5 | ): Promise {
6 | const canvas = document.createElement('canvas');
7 | const ctx = canvas.getContext('2d', { willReadFrequently: true });
8 |
9 | if (!ctx) {
10 | return [];
11 | }
12 |
13 | canvas.width = video.videoWidth;
14 | canvas.height = video.videoHeight;
15 |
16 | const numberOfFrames = 5;
17 |
18 | const timeStamps = linSpace(
19 | video.duration * 0.2,
20 | video.duration * 0.8,
21 | numberOfFrames + 2,
22 | )
23 | .map((n) => Math.floor(n))
24 | .slice(1, -1);
25 |
26 | const videoClone = document.createElement('video');
27 | videoClone.crossOrigin = 'anonymous';
28 | videoClone.src = video.src;
29 | videoClone.muted = true;
30 | videoClone.pause();
31 |
32 | const promises = timeStamps.map((timeStamp) => {
33 | return new Promise((resolve) => {
34 | const onSeeked = () => {
35 | videoClone.removeEventListener('seeked', onSeeked);
36 | ctx.drawImage(videoClone, 0, 0);
37 | resolve(ctx.getImageData(0, 0, canvas.width, canvas.height));
38 | };
39 |
40 | videoClone.addEventListener('seeked', onSeeked);
41 | videoClone.currentTime = timeStamp;
42 | });
43 | });
44 |
45 | const frames = await Promise.all(promises);
46 | return frames;
47 | }
48 |
--------------------------------------------------------------------------------
/packages/app/src/hooks/useXRSession.ts:
--------------------------------------------------------------------------------
1 | import { atom, useAtom } from 'jotai';
2 | import { useCallback, useEffect } from 'react';
3 |
4 | const xrSupportedAtom = atom(false);
5 | const xrSessionAtom = atom(null);
6 |
7 | function checkForXRSupport() {
8 | if (!navigator.xr) {
9 | return Promise.resolve(false);
10 | }
11 | return navigator.xr.isSessionSupported('immersive-vr');
12 | }
13 |
14 | export const useXRSession = () => {
15 | const [xrSupported, setXrSupported] = useAtom(xrSupportedAtom);
16 | const [xrSession, setXrSession] = useAtom(xrSessionAtom);
17 |
18 | useEffect(() => {
19 | const onDevicechange = () => {
20 | void checkForXRSupport().then(setXrSupported);
21 | };
22 |
23 | navigator.xr?.addEventListener('devicechange', onDevicechange);
24 |
25 | onDevicechange();
26 |
27 | return () => {
28 | navigator.xr?.removeEventListener('devicechange', onDevicechange);
29 | };
30 | }, [setXrSupported]);
31 |
32 | useEffect(() => {
33 | const onXrSessionEnd = () => {
34 | setXrSession(null);
35 | };
36 |
37 | if (xrSession) {
38 | xrSession.addEventListener('end', onXrSessionEnd);
39 | }
40 |
41 | return () => {
42 | xrSession?.removeEventListener('end', onXrSessionEnd);
43 | };
44 | }, [setXrSession, xrSession]);
45 |
46 | const requestXrSession = useCallback(() => {
47 | if (xrSupported && !xrSession) {
48 | void navigator.xr?.requestSession('immersive-vr').then((session) => {
49 | setXrSession(session);
50 | });
51 | }
52 | }, [setXrSession, xrSession, xrSupported]);
53 |
54 | return [xrSupported, xrSession, requestXrSession] as const;
55 | };
56 |
--------------------------------------------------------------------------------
/packages/app/src/worker/videoRecognition.worker/videoRecognition.spec.ts:
--------------------------------------------------------------------------------
1 | import { detectLayout } from './videoRecognition';
2 | import type { Layout } from '@vr-viewer/player';
3 |
4 | const IMAGE_WIDTH = 4;
5 | const IMAGE_HEIGHT = 4;
6 |
7 | describe('Test video recognition', () => {
8 | test('Detect top/bottom video', () => {
9 | const pixels = new Array(IMAGE_HEIGHT).fill(
10 | [
11 | ...new Array(IMAGE_WIDTH / 2).fill([0, 0, 0, 1]),
12 | ...new Array(IMAGE_WIDTH / 2).fill([1, 1, 1, 1]),
13 | ].flat(),
14 | );
15 |
16 | const imageData = new ImageData(
17 | Uint8ClampedArray.from(pixels.flat()),
18 | IMAGE_WIDTH,
19 | IMAGE_HEIGHT,
20 | );
21 |
22 | expect(detectLayout([imageData])).toBe('stereoTopBottom');
23 | });
24 |
25 | test('Detect left/right video', () => {
26 | const pixels = [
27 | ...new Array(IMAGE_WIDTH * (IMAGE_HEIGHT / 2)).fill([
28 | 0, 0, 0, 1,
29 | ]),
30 | ...new Array(IMAGE_WIDTH * (IMAGE_HEIGHT / 2)).fill([
31 | 1, 1, 1, 1,
32 | ]),
33 | ];
34 |
35 | const imageData = new ImageData(
36 | Uint8ClampedArray.from(pixels.flat()),
37 | IMAGE_WIDTH,
38 | IMAGE_HEIGHT,
39 | );
40 |
41 | expect(detectLayout([imageData])).toBe('stereoLeftRight');
42 | });
43 |
44 | test('Detect mono video', () => {
45 | const pixels = new Array((IMAGE_WIDTH * IMAGE_HEIGHT) / 2).fill(
46 | [...[0, 0, 0, 1], ...[1, 1, 1, 1]].flat(),
47 | );
48 |
49 | const imageData = new ImageData(
50 | Uint8ClampedArray.from(pixels.flat()),
51 | IMAGE_WIDTH,
52 | IMAGE_HEIGHT,
53 | );
54 |
55 | expect(detectLayout([imageData])).toBe('mono');
56 | });
57 | });
58 |
--------------------------------------------------------------------------------
/packages/app/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 | VR Player | Play 360° 3D VR Videos in Your Browser
16 |
20 |
24 |
28 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
--------------------------------------------------------------------------------
/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "root": true,
3 | "env": {
4 | "es6": true,
5 | },
6 | "parser": "@typescript-eslint/parser",
7 | "parserOptions": {
8 | "ecmaVersion": 2017,
9 | },
10 | "plugins": ["@typescript-eslint", "prettier", "sort-imports-es6-autofix"],
11 | "extends": [
12 | "eslint:recommended",
13 | "plugin:import/recommended",
14 | "plugin:import/typescript",
15 | "plugin:@typescript-eslint/recommended",
16 | "plugin:@typescript-eslint/recommended-requiring-type-checking",
17 | "plugin:prettier/recommended",
18 | ],
19 | "settings": {
20 | "import/parsers": {
21 | "@typescript-eslint/parser": [".ts", ".tsx"],
22 | },
23 | "import/resolver": {
24 | "typescript": {
25 | "alwaysTryTypes": true, // always try to resolve types under `@types` directory even it doesn't contain any source code, like `@types/unist`
26 |
27 | // use a glob pattern
28 | "project": "packages/*/tsconfig.json",
29 | },
30 | },
31 | },
32 | "rules": {
33 | "@typescript-eslint/no-explicit-any": [
34 | "error",
35 | {
36 | "ignoreRestArgs": true,
37 | },
38 | ],
39 | "import/prefer-default-export": "off",
40 | "import/no-default-export": "error",
41 | "object-curly-newline": "off",
42 | "no-void": [
43 | "error",
44 | {
45 | "allowAsStatement": true,
46 | },
47 | ],
48 | "react/require-default-props": "off",
49 | "react/jsx-props-no-spreading": "off",
50 | "@typescript-eslint/consistent-type-imports": [
51 | "error",
52 | {
53 | "prefer": "type-imports",
54 | "disallowTypeAnnotations": false,
55 | },
56 | ],
57 | "import/order": "off",
58 | "sort-imports-es6-autofix/sort-imports-es6": [
59 | "error",
60 | {
61 | "ignoreCase": false,
62 | "ignoreMemberSort": false,
63 | "memberSyntaxSortOrder": ["none", "all", "multiple", "single"],
64 | },
65 | ],
66 | },
67 | }
68 |
--------------------------------------------------------------------------------
/packages/app/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@vr-viewer/app",
3 | "version": "0.1.0",
4 | "private": true,
5 | "dependencies": {
6 | "@emotion/react": "^11.13.3",
7 | "@heroicons/react": "^2.1.5",
8 | "@testing-library/jest-dom": "^6.5.0",
9 | "@testing-library/react": "^16.0.1",
10 | "@testing-library/user-event": "^14.5.2",
11 | "@types/jest": "^29.5.13",
12 | "@types/node": "^22.7.4",
13 | "@types/react": "^18.3.10",
14 | "@types/react-dom": "^18.3.0",
15 | "@types/webxr": "^0.5.20",
16 | "@vercel/analytics": "^1.3.1",
17 | "@vr-viewer/player": "*",
18 | "autoprefixer": "^10.4.20",
19 | "clsx": "^2.1.1",
20 | "comlink": "^4.4.1",
21 | "jest-canvas-mock": "^2.5.2",
22 | "jotai": "^2.10.0",
23 | "jotai-devtools": "^0.10.1",
24 | "postcss": "^8.4.47",
25 | "react": "^18.3.1",
26 | "react-dom": "^18.3.1",
27 | "react-dropzone": "^14.2.3",
28 | "react-hot-toast": "^2.4.1",
29 | "react-scripts": "5.0.1",
30 | "tailwindcss": "^3.4.13",
31 | "typescript": "^5.4.3",
32 | "web-vitals": "^4.2.3",
33 | "webxr-polyfill": "^2.0.3",
34 | "workbox-background-sync": "^7.1.0",
35 | "workbox-broadcast-update": "^7.1.0",
36 | "workbox-cacheable-response": "^7.1.0",
37 | "workbox-core": "^7.1.0",
38 | "workbox-expiration": "^7.1.0",
39 | "workbox-navigation-preload": "^7.1.0",
40 | "workbox-precaching": "^7.1.0",
41 | "workbox-range-requests": "^7.1.0",
42 | "workbox-routing": "^7.1.0",
43 | "workbox-strategies": "^7.1.0",
44 | "workbox-streams": "^7.1.0"
45 | },
46 | "scripts": {
47 | "start": "react-scripts start",
48 | "build": "react-scripts build",
49 | "test": "react-scripts test",
50 | "lint": "eslint ."
51 | },
52 | "eslintConfig": {
53 | "extends": [
54 | "react-app",
55 | "react-app/jest"
56 | ]
57 | },
58 | "browserslist": {
59 | "production": [
60 | ">0.2%",
61 | "not dead",
62 | "not op_mini all"
63 | ],
64 | "development": [
65 | "last 1 chrome version",
66 | "last 1 firefox version",
67 | "last 1 safari version"
68 | ]
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/packages/player/src/renderer/debugRenderer.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-underscore-dangle */
2 | import { Renderer } from './renderer';
3 | import { mat4 } from 'gl-matrix';
4 | import type { Format, Layout } from '../types';
5 | import type { RenderProps } from './renderProps';
6 | import type { Texture2DOptions } from 'regl';
7 |
8 | export class DebugRenderer extends Renderer {
9 | private raf = 0;
10 |
11 | constructor(
12 | private readonly video: HTMLVideoElement,
13 | canvas: HTMLCanvasElement,
14 | layout: Layout,
15 | flipLayout: boolean,
16 | format: Format,
17 | private readonly view: 'left' | 'right',
18 | ) {
19 | super(canvas, layout, flipLayout, format);
20 | }
21 |
22 | protected stopDrawLoop(): void {
23 | window.cancelAnimationFrame(this.raf);
24 | }
25 |
26 | protected async startDrawLoop(): Promise {
27 | const textureProps: Texture2DOptions = { data: this.video, flipY: true };
28 | const texture = this.regl.texture(textureProps);
29 |
30 | const model = this.getModelMatrix(this.video);
31 |
32 | const view = mat4.lookAt(mat4.create(), [0, 0, 0], [0, 0, -1], [0, 1, 0]);
33 |
34 | const projection = mat4.perspective(
35 | mat4.create(),
36 | Math.PI / 2,
37 | this.canvas.width / this.canvas.height,
38 | 0.01,
39 | Infinity,
40 | );
41 |
42 | const offsets = this.getTexCoordScaleOffsets();
43 | const offset = this.view === 'left' ? offsets[0] : offsets[1];
44 |
45 | const drawLoop = () => {
46 | this.regl.clear({ color: [0, 0, 0, 1], depth: 1 });
47 |
48 | const props: RenderProps = {
49 | model,
50 | view,
51 | projection,
52 | texture: texture.subimage(textureProps),
53 | viewport: {
54 | x: 0,
55 | y: 0,
56 | width: this.canvas.width,
57 | height: this.canvas.height,
58 | },
59 | texCoordScaleOffset: offset,
60 | };
61 | this.cmdRender(props);
62 |
63 | this.raf = window.requestAnimationFrame(drawLoop);
64 | };
65 |
66 | this.raf = window.requestAnimationFrame(drawLoop);
67 |
68 | return Promise.resolve();
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/packages/app/README.md:
--------------------------------------------------------------------------------
1 | # Getting Started with Create React App
2 |
3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
4 |
5 | ## Available Scripts
6 |
7 | In the project directory, you can run:
8 |
9 | ### `npm start`
10 |
11 | Runs the app in the development mode.\
12 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
13 |
14 | The page will reload if you make edits.\
15 | You will also see any lint errors in the console.
16 |
17 | ### `npm test`
18 |
19 | Launches the test runner in the interactive watch mode.\
20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
21 |
22 | ### `npm run build`
23 |
24 | Builds the app for production to the `build` folder.\
25 | It correctly bundles React in production mode and optimizes the build for the best performance.
26 |
27 | The build is minified and the filenames include the hashes.\
28 | Your app is ready to be deployed!
29 |
30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
31 |
32 | ### `npm run eject`
33 |
34 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!**
35 |
36 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
37 |
38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own.
39 |
40 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it.
41 |
42 | ## Learn More
43 |
44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
45 |
46 | To learn React, check out the [React documentation](https://reactjs.org/).
47 |
--------------------------------------------------------------------------------
/packages/app/src/logo.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/packages/player/src/primitive/sphere.ts:
--------------------------------------------------------------------------------
1 | import { mat4, vec2, vec3 } from 'gl-matrix';
2 | import type { Primitive } from './primitive';
3 |
4 | const up = vec3.fromValues(0, 1, 0);
5 |
6 | export function sphere({
7 | radius = 1,
8 | segments = 32,
9 | halfSphere = false,
10 | }): Primitive {
11 | const indices = [] as vec3[];
12 | const positions = [] as vec3[];
13 | const normals = [] as vec3[];
14 | const uvs = [] as vec2[];
15 |
16 | const tmpVec3 = vec3.create();
17 | const tmpMatRotZ = mat4.create();
18 | const tmpMatRotY = mat4.create();
19 |
20 | const totalZRotationSteps = 2 + segments;
21 | const totalYRotationSteps = 2 * totalZRotationSteps;
22 |
23 | for (
24 | let zRotationStep = 0;
25 | zRotationStep <= totalZRotationSteps;
26 | zRotationStep += 1
27 | ) {
28 | const normalizedZ = zRotationStep / totalZRotationSteps;
29 | const angleZ = normalizedZ * Math.PI;
30 |
31 | for (
32 | let yRotationStep = 0;
33 | yRotationStep <= totalYRotationSteps;
34 | yRotationStep += 1
35 | ) {
36 | const normalizedY = yRotationStep / totalYRotationSteps;
37 | const angleY = normalizedY * Math.PI * 2;
38 |
39 | mat4.identity(tmpMatRotZ);
40 | mat4.rotateZ(tmpMatRotZ, tmpMatRotZ, -angleZ);
41 |
42 | mat4.identity(tmpMatRotY);
43 | mat4.rotateY(tmpMatRotY, tmpMatRotY, halfSphere ? angleY / 2 : angleY);
44 |
45 | vec3.transformMat4(tmpVec3, up, tmpMatRotZ);
46 | vec3.transformMat4(tmpVec3, tmpVec3, tmpMatRotY);
47 |
48 | vec3.scale(tmpVec3, tmpVec3, -radius);
49 | positions.push(vec3.clone(tmpVec3));
50 |
51 | vec3.normalize(tmpVec3, tmpVec3);
52 | normals.push(vec3.clone(tmpVec3));
53 |
54 | uvs.push(vec2.fromValues(1 - normalizedY, normalizedZ));
55 | }
56 |
57 | if (zRotationStep > 0) {
58 | const verticesCount = positions.length;
59 |
60 | for (
61 | let firstIndex = verticesCount - 2 * (totalYRotationSteps + 1);
62 | firstIndex + totalYRotationSteps + 2 < verticesCount;
63 | firstIndex += 1
64 | ) {
65 | indices.push(
66 | vec3.fromValues(
67 | firstIndex,
68 | firstIndex + 1,
69 | firstIndex + totalYRotationSteps + 1,
70 | ),
71 | );
72 | indices.push(
73 | vec3.fromValues(
74 | firstIndex + totalYRotationSteps + 1,
75 | firstIndex + 1,
76 | firstIndex + totalYRotationSteps + 2,
77 | ),
78 | );
79 | }
80 | }
81 | }
82 |
83 | return {
84 | indices,
85 | positions,
86 | normals,
87 | uvs,
88 | };
89 | }
90 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | ###############################################################################
2 | # Set default behavior to automatically normalize line endings.
3 | ###############################################################################
4 | * text=auto
5 |
6 | # Shell scripts should always use line feed not crlf
7 | *.sh text eol=lf
8 |
9 | ###############################################################################
10 | # Set the merge driver for project and solution files
11 | #
12 | # Merging from the command prompt will add diff markers to the files if there
13 | # are conflicts (Merging from VS is not affected by the settings below, in VS
14 | # the diff markers are never inserted). Diff markers may cause the following
15 | # file extensions to fail to load in VS. An alternative would be to treat
16 | # these files as binary and thus will always conflict and require user
17 | # intervention with every merge. To do so, just uncomment the entries below
18 | ###############################################################################
19 | *.js text
20 | *.ts text
21 | *.json text
22 | *.resjson text
23 | *.htm text
24 | *.html text
25 | *.xml text
26 | *.txt text
27 | *.ini text
28 | *.inc text
29 | #*.sln merge=binary
30 | #*.csproj merge=binary
31 | #*.vbproj merge=binary
32 | #*.vcxproj merge=binary
33 | #*.vcproj merge=binary
34 | #*.dbproj merge=binary
35 | #*.fsproj merge=binary
36 | #*.lsproj merge=binary
37 | #*.wixproj merge=binary
38 | #*.modelproj merge=binary
39 | #*.sqlproj merge=binary
40 | #*.wwaproj merge=binary
41 |
42 | ###############################################################################
43 | # behavior for image files
44 | #
45 | # image files are treated as binary by default.
46 | ###############################################################################
47 | *.png binary
48 | *.jpg binary
49 | *.jpeg binary
50 | *.gif binary
51 | *.ico binary
52 | *.mov binary
53 | *.mp4 binary
54 | *.mp3 binary
55 | *.flv binary
56 | *.fla binary
57 | *.swf binary
58 | *.gz binary
59 | *.zip binary
60 | *.7z binary
61 | *.ttf binary
62 |
63 | ###############################################################################
64 | # diff behavior for common document formats
65 | #
66 | # Convert binary document formats to text before diffing them. This feature
67 | # is only available from the command line. Turn it on by uncommenting the
68 | # entries below.
69 | ###############################################################################
70 | *.doc diff=astextplain
71 | *.DOC diff=astextplain
72 | *.docx diff=astextplain
73 | *.DOCX diff=astextplain
74 | *.dot diff=astextplain
75 | *.DOT diff=astextplain
76 | *.pdf diff=astextplain
77 | *.PDF diff=astextplain
78 | *.rtf diff=astextplain
79 | *.RTF diff=astextplain
80 |
--------------------------------------------------------------------------------
/packages/app/src/service-worker.ts:
--------------------------------------------------------------------------------
1 | ///
2 | /* eslint-disable no-restricted-globals */
3 |
4 | // This service worker can be customized!
5 | // See https://developers.google.com/web/tools/workbox/modules
6 | // for the list of available Workbox modules, or add any other
7 | // code you'd like.
8 | // You can also remove this file if you'd prefer not to use a
9 | // service worker, and the Workbox build step will be skipped.
10 |
11 | import { CacheFirst } from 'workbox-strategies';
12 | import { CacheableResponsePlugin } from 'workbox-cacheable-response';
13 | import { ExpirationPlugin } from 'workbox-expiration';
14 | import { clientsClaim } from 'workbox-core';
15 | import { createHandlerBoundToURL, precacheAndRoute } from 'workbox-precaching';
16 | import { registerRoute } from 'workbox-routing';
17 |
18 | declare const self: ServiceWorkerGlobalScope;
19 |
20 | clientsClaim();
21 |
22 | // Precache all of the assets generated by your build process.
23 | // Their URLs are injected into the manifest variable below.
24 | // This variable must be present somewhere in your service worker file,
25 | // even if you decide not to use precaching. See https://cra.link/PWA
26 | // eslint-disable-next-line no-underscore-dangle
27 | precacheAndRoute(self.__WB_MANIFEST);
28 |
29 | // Set up App Shell-style routing, so that all navigation requests
30 | // are fulfilled with your index.html shell. Learn more at
31 | // https://developers.google.com/web/fundamentals/architecture/app-shell
32 | const fileExtensionRegexp = /\/[^/?]+\.[^/]+$/;
33 | registerRoute(
34 | // Return false to exempt requests from being fulfilled by index.html.
35 | ({ request, url }: { request: Request; url: URL }) => {
36 | // If this isn't a navigation, skip.
37 | if (request.mode !== 'navigate') {
38 | return false;
39 | }
40 |
41 | // If this is a URL that starts with /_, skip.
42 | if (url.pathname.startsWith('/_')) {
43 | return false;
44 | }
45 |
46 | // If this looks like a URL for a resource, because it contains
47 | // a file extension, skip.
48 | if (url.pathname.match(fileExtensionRegexp)) {
49 | return false;
50 | }
51 |
52 | // Return true to signal that we want to use the handler.
53 | return true;
54 | },
55 | createHandlerBoundToURL('/index.html'),
56 | );
57 |
58 | registerRoute(
59 | ({ url }) =>
60 | url.origin === self.location.origin &&
61 | /.(?:png|gif|jpg|jpeg|webp|svg)$/.test(url.pathname),
62 | new CacheFirst({
63 | cacheName: 'images',
64 | plugins: [
65 | new CacheableResponsePlugin({
66 | statuses: [0, 200],
67 | }),
68 | new ExpirationPlugin({
69 | maxEntries: 20,
70 | maxAgeSeconds: 12 * 60 * 60,
71 | }),
72 | ],
73 | }),
74 | );
75 |
76 | // This allows the web app to trigger skipWaiting via
77 | // registration.waiting.postMessage({type: 'SKIP_WAITING'})
78 | self.addEventListener('message', (event) => {
79 | const { data } = event as { data: { type: 'SKIP_WAITING' } };
80 |
81 | if (data && data.type === 'SKIP_WAITING') {
82 | void self.skipWaiting();
83 | }
84 | });
85 |
86 | // Any other custom service worker logic can go here.
87 |
--------------------------------------------------------------------------------
/packages/app/public/safari-pinned-tab.svg:
--------------------------------------------------------------------------------
1 |
2 |
4 |
51 |
--------------------------------------------------------------------------------
/packages/player/src/renderer/vrRenderer.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-underscore-dangle */
2 | import { Renderer } from './renderer';
3 | import { mat4, vec3 } from 'gl-matrix';
4 | import type { Format, Layout } from '../types';
5 | import type { RenderProps } from './renderProps';
6 | import type { Texture2DOptions } from 'regl';
7 |
8 | export class VrRenderer extends Renderer {
9 | private raf = 0;
10 |
11 | constructor(
12 | private readonly xrSession: XRSession,
13 | private readonly video: HTMLVideoElement,
14 | canvas: HTMLCanvasElement,
15 | layout: Layout,
16 | flipLayout: boolean,
17 | format: Format,
18 | ) {
19 | super(canvas, layout, flipLayout, format);
20 | }
21 |
22 | protected stopDrawLoop(): void {
23 | window.cancelAnimationFrame(this.raf);
24 | }
25 |
26 | protected async startDrawLoop(): Promise {
27 | // **** First hack to get this to work with regl:
28 | await this.regl._gl.makeXRCompatible();
29 | await this.xrSession.updateRenderState({
30 | baseLayer: new XRWebGLLayer(this.xrSession, this.regl._gl),
31 | depthFar: this.xrSession.renderState.depthFar,
32 | depthNear: this.xrSession.renderState.depthNear,
33 | });
34 |
35 | const textureProps: Texture2DOptions = { data: this.video, flipY: true };
36 | const texture = this.regl.texture(textureProps);
37 |
38 | const model = this.getModelMatrix(this.video);
39 |
40 | const view = mat4.create();
41 |
42 | const offsets = this.getTexCoordScaleOffsets();
43 | const xrReferenceSpace =
44 | await this.xrSession.requestReferenceSpace('local');
45 |
46 | const drawLoop: XRFrameRequestCallback = (
47 | _time: DOMHighResTimeStamp,
48 | xrFrame: XRFrame,
49 | ) => {
50 | const glLayer = this.xrSession.renderState.baseLayer;
51 | const pose = xrFrame.getViewerPose(xrReferenceSpace);
52 |
53 | if (!glLayer || !pose) {
54 | return;
55 | }
56 |
57 | // **** Second hack to get this to work with regl. Bind the framebuffer and clear it before
58 | // **** rendering to it. Note that this is not a regl framebuffer, it's just a WebGL framebuffer
59 | // **** ID handed to us by WebXR.
60 | this.regl._gl.bindFramebuffer(
61 | this.regl._gl.FRAMEBUFFER,
62 | glLayer.framebuffer,
63 | );
64 | this.regl._gl.clearColor(0, 0, 0, 1);
65 | this.regl._gl.clear(
66 | // eslint-disable-next-line no-bitwise
67 | this.regl._gl.DEPTH_BUFFER_BIT | this.regl._gl.COLOR_BUFFER_BIT,
68 | );
69 |
70 | // Render each eye.
71 | pose.views.forEach((poseView) => {
72 | const { position } = poseView.transform;
73 |
74 | const viewport = glLayer.getViewport(poseView);
75 |
76 | if (viewport === undefined) {
77 | throw new Error('Viewport is undefined');
78 | }
79 |
80 | const props: RenderProps = {
81 | model,
82 | view: mat4.translate(
83 | view,
84 | poseView.transform.inverse.matrix,
85 | vec3.fromValues(position.x, position.y, position.z),
86 | ),
87 | projection: poseView.projectionMatrix,
88 | texture: texture.subimage(textureProps),
89 | viewport,
90 | texCoordScaleOffset:
91 | poseView.eye === 'left' ? offsets[0] : offsets[1],
92 | };
93 | this.cmdRender(props);
94 | });
95 |
96 | this.raf = this.xrSession.requestAnimationFrame(drawLoop);
97 | };
98 |
99 | this.raf = this.xrSession.requestAnimationFrame(drawLoop);
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/packages/app/src/worker/videoRecognition.worker/videoRecognition.ts:
--------------------------------------------------------------------------------
1 | import { linSpace, nWise } from 'helper/util';
2 | import type { Format, Layout } from '@vr-viewer/player';
3 |
4 | const PIXEL_RGBA_SIZE = 4;
5 |
6 | function getPixelDiff(a: number[][], b: number[][]) {
7 | const diffs = a.map((pixel, index) => {
8 | const rDiff = pixel[0] - b[index][0];
9 | const gDiff = pixel[1] - b[index][1];
10 | const bDiff = pixel[2] - b[index][2];
11 | return Math.sqrt(rDiff ** 2 + gDiff ** 2 + bDiff ** 2);
12 | });
13 | const average = diffs.reduce((_a, _b) => _a + _b, 0) / diffs.length;
14 | return average;
15 | }
16 |
17 | function getPixelRow(imageData: ImageData, row: number) {
18 | const rowData = imageData.data.slice(
19 | row * imageData.width * PIXEL_RGBA_SIZE,
20 | (row + 1) * imageData.width * PIXEL_RGBA_SIZE,
21 | );
22 | return Array.from(nWise(PIXEL_RGBA_SIZE, rowData));
23 | }
24 |
25 | function getPixelColumn(imageData: ImageData, column: number) {
26 | const columnPixels = [] as number[][];
27 |
28 | for (
29 | let i = column * PIXEL_RGBA_SIZE;
30 | i < imageData.height * imageData.width * PIXEL_RGBA_SIZE;
31 | i += imageData.width * PIXEL_RGBA_SIZE
32 | ) {
33 | columnPixels.push(Array.from(imageData.data.slice(i, i + PIXEL_RGBA_SIZE)));
34 | }
35 |
36 | return columnPixels;
37 | }
38 |
39 | export function detectLayout(frames: ImageData[]): Layout {
40 | const frameVertMirrorValues = frames.map((frame) => {
41 | const numRows = 30;
42 | const rowNumbers = linSpace(0, frame.height - 1, numRows).map((n) =>
43 | Math.floor(n),
44 | );
45 |
46 | const rows = rowNumbers.map((rowNumber) => getPixelRow(frame, rowNumber));
47 |
48 | const diffs = rows.map((row) => {
49 | return getPixelDiff(
50 | row.slice(0, frame.width / 2),
51 | row.slice(-(frame.width / 2)),
52 | );
53 | });
54 |
55 | const avgDiff = diffs.reduce((_a, _b) => _a + _b, 0) / diffs.length;
56 | return avgDiff;
57 | });
58 |
59 | const avgFrameVertMirrorValues =
60 | frameVertMirrorValues.reduce((a, b) => a + b, 0) /
61 | frameVertMirrorValues.length;
62 |
63 | const frameHorizontalMirrorValues = frames.map((frame) => {
64 | const numColumns = 30;
65 | const columnNumbers = linSpace(0, frame.width - 1, numColumns).map((n) =>
66 | Math.floor(n),
67 | );
68 |
69 | const columns = columnNumbers.map((columnNumber) =>
70 | getPixelColumn(frame, columnNumber),
71 | );
72 |
73 | const diffs = columns.map((column) => {
74 | return getPixelDiff(
75 | column.slice(0, frame.height / 2),
76 | column.slice(-(frame.height / 2)),
77 | );
78 | });
79 |
80 | const avgDiff = diffs.reduce((_a, _b) => _a + _b, 0) / diffs.length;
81 | return avgDiff;
82 | });
83 |
84 | const avgFrameHorizontalMirrorValues =
85 | frameHorizontalMirrorValues.reduce((a, b) => a + b, 0) /
86 | frameHorizontalMirrorValues.length;
87 |
88 | if (avgFrameVertMirrorValues < avgFrameHorizontalMirrorValues / 1.5) {
89 | return 'stereoLeftRight';
90 | }
91 |
92 | if (avgFrameHorizontalMirrorValues < avgFrameVertMirrorValues / 1.5) {
93 | return 'stereoTopBottom';
94 | }
95 |
96 | return 'mono';
97 | }
98 |
99 | export function detectFormat(frames: ImageData[]): Format {
100 | const frame = frames[0];
101 |
102 | const topRowPixels = getPixelRow(frame, 0);
103 | const bottomRowPixels = getPixelRow(frame, frame.height - 1);
104 |
105 | const topRowDiff = topRowPixels.map((pixel, i) => {
106 | if (i === topRowPixels.length - 1) {
107 | return 0;
108 | }
109 | return getPixelDiff([pixel], [topRowPixels[i + 1]]);
110 | });
111 | const avgTopRowDiff =
112 | topRowDiff.reduce((a, b) => a + b, 0) / topRowDiff.length;
113 |
114 | const bottomRowDiff = bottomRowPixels.map((pixel, i) => {
115 | if (i === bottomRowPixels.length - 1) {
116 | return 0;
117 | }
118 |
119 | return getPixelDiff([pixel], [bottomRowPixels[i + 1]]);
120 | });
121 | const avgBottomRowDiff =
122 | bottomRowDiff.reduce((a, b) => a + b, 0) / bottomRowDiff.length;
123 |
124 | if (avgTopRowDiff + avgBottomRowDiff < 1) {
125 | const leftColumn = getPixelColumn(frame, 0);
126 | const rightColumn = getPixelColumn(frame, frame.width - 1);
127 |
128 | const leftRightDiff = getPixelDiff(leftColumn, rightColumn);
129 |
130 | if (leftRightDiff < 10) {
131 | return '360';
132 | }
133 |
134 | return '180';
135 | }
136 |
137 | return 'screen';
138 | }
139 |
--------------------------------------------------------------------------------
/packages/player/src/renderer/renderer.ts:
--------------------------------------------------------------------------------
1 | import { mat4 } from 'gl-matrix';
2 | import { sphere } from '../primitive';
3 | import { square } from '../primitive/square';
4 | import reglInit from 'regl';
5 | import type { Format, Layout } from '../types';
6 | import type { Primitive } from '../primitive';
7 | import type { Regl } from 'regl';
8 | import type { RenderProps } from './renderProps';
9 |
10 | export abstract class Renderer {
11 | protected abstract startDrawLoop(): Promise;
12 | protected abstract stopDrawLoop(): void;
13 |
14 | public async start() {
15 | await this.startDrawLoop();
16 | }
17 |
18 | public stop() {
19 | this.stopDrawLoop();
20 | this.regl.destroy();
21 | }
22 |
23 | protected readonly regl: Regl;
24 |
25 | protected readonly cmdRender: reglInit.DrawCommand<
26 | reglInit.DefaultContext,
27 | RenderProps
28 | >;
29 |
30 | constructor(
31 | protected readonly canvas: HTMLCanvasElement,
32 | protected readonly layout: Layout,
33 | protected readonly flipLayout: boolean,
34 | protected readonly format: Format,
35 | ) {
36 | const mesh = this.getMesh();
37 |
38 | this.regl = reglInit({ pixelRatio: 1, canvas: this.canvas });
39 |
40 | this.cmdRender = this.regl({
41 | vert: `
42 | precision highp float;
43 |
44 | attribute vec3 position;
45 | attribute vec2 uv;
46 |
47 | uniform vec4 texCoordScaleOffset;
48 | uniform mat4 model;
49 | uniform mat4 view;
50 | uniform mat4 projection;
51 |
52 | varying vec2 mappedUv;
53 |
54 | void main() {
55 | gl_Position = projection * view * model * vec4(position, 1);
56 | mappedUv = uv * texCoordScaleOffset.xy + texCoordScaleOffset.zw;
57 | }
58 | `,
59 | frag: `
60 | precision highp float;
61 |
62 | uniform sampler2D texture;
63 |
64 | varying vec2 mappedUv;
65 |
66 | void main() {
67 | gl_FragColor = texture2D(texture, mappedUv);
68 | }
69 | `,
70 | attributes: {
71 | position: mesh.positions,
72 | uv: mesh.uvs,
73 | },
74 | // TODO: https://github.com/regl-project/regl/pull/632
75 | uniforms: {
76 | model: this.regl.prop('model'),
77 | view: this.regl.prop('view'),
78 | projection: this.regl.prop('projection'),
79 | texture: this.regl.prop('texture'),
80 | texCoordScaleOffset: this.regl.prop(
81 | 'texCoordScaleOffset',
82 | ),
83 | },
84 | viewport: this.regl.prop('viewport'),
85 | elements: mesh.indices,
86 | cull: {
87 | enable: true,
88 | face: 'front',
89 | },
90 | });
91 | }
92 |
93 | private getMesh(): Primitive {
94 | switch (this.format) {
95 | case '360':
96 | return sphere({ radius: 1, segments: 32 });
97 | case '180': {
98 | return sphere({ radius: 1, segments: 16, halfSphere: true });
99 | }
100 | case 'screen':
101 | // falls through
102 | default:
103 | return square({ scale: 1 });
104 | }
105 | }
106 |
107 | private getAspectRation(video: HTMLVideoElement) {
108 | switch (this.layout) {
109 | case 'stereoLeftRight':
110 | return (video.videoWidth * 0.5) / video.videoHeight;
111 | case 'stereoTopBottom':
112 | return (video.videoWidth / video.videoHeight) * 0.5;
113 | case 'mono':
114 | // falls through
115 | default:
116 | return video.videoWidth / video.videoHeight;
117 | }
118 | }
119 |
120 | protected getModelMatrix(video: HTMLVideoElement) {
121 | const aspectRatio = this.getAspectRation(video);
122 |
123 | const model = mat4.create();
124 | // rotate model 180 deg to flip z axis as WebXR looks towards -z
125 | // https://developer.mozilla.org/en-US/docs/Web/API/WebXR_Device_API/Geometry
126 | mat4.rotateY(model, model, Math.PI);
127 |
128 | if (this.format === 'screen') {
129 | const screenHeight = 1;
130 |
131 | // scale according to aspect ratio
132 | mat4.scale(model, model, [screenHeight * aspectRatio, screenHeight, 1]);
133 | // move screen back a bit
134 | mat4.translate(model, model, [0, 0, screenHeight]);
135 | }
136 |
137 | if (this.format === '360') {
138 | // rotate model 90 deg to look at the center of the video
139 | mat4.rotateY(model, model, -Math.PI / 2);
140 | }
141 |
142 | return model;
143 | }
144 |
145 | protected getTexCoordScaleOffsets() {
146 | let offsets;
147 | switch (this.layout) {
148 | case 'stereoLeftRight':
149 | offsets = [
150 | new Float32Array([0.5, 1.0, 0.0, 0.0]),
151 | new Float32Array([0.5, 1.0, 0.5, 0.0]),
152 | ];
153 | break;
154 | case 'stereoTopBottom':
155 | offsets = [
156 | new Float32Array([1.0, 0.5, 0.0, 0.0]),
157 | new Float32Array([1.0, 0.5, 0.0, 0.5]),
158 | ];
159 | break;
160 | case 'mono':
161 | // falls through
162 | default:
163 | offsets = [
164 | new Float32Array([1.0, 1.0, 0.0, 0.0]),
165 | new Float32Array([1.0, 1.0, 0.0, 0.0]),
166 | ];
167 | }
168 | if (this.flipLayout) {
169 | return offsets.reverse();
170 | }
171 | return offsets;
172 | }
173 | }
174 |
--------------------------------------------------------------------------------
/packages/app/src/serviceWorkerRegistration.ts:
--------------------------------------------------------------------------------
1 | /* eslint-disable no-console */
2 | // This optional code is used to register a service worker.
3 | // register() is not called by default.
4 |
5 | // This lets the app load faster on subsequent visits in production, and gives
6 | // it offline capabilities. However, it also means that developers (and users)
7 | // will only see deployed updates on subsequent visits to a page, after all the
8 | // existing tabs open on the page have been closed, since previously cached
9 | // resources are updated in the background.
10 |
11 | // To learn more about the benefits of this model and instructions on how to
12 | // opt-in, read https://cra.link/PWA
13 |
14 | const isLocalhost = Boolean(
15 | window.location.hostname === 'localhost' ||
16 | // [::1] is the IPv6 localhost address.
17 | window.location.hostname === '[::1]' ||
18 | // 127.0.0.0/8 are considered localhost for IPv4.
19 | window.location.hostname.match(
20 | /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/,
21 | ),
22 | );
23 |
24 | type Config = {
25 | onSuccess?: (registration: ServiceWorkerRegistration) => void;
26 | onUpdate?: (registration: ServiceWorkerRegistration) => void;
27 | };
28 |
29 | function registerValidSW(swUrl: string, config?: Config) {
30 | navigator.serviceWorker
31 | .register(swUrl)
32 | .then((registration) => {
33 | // eslint-disable-next-line no-param-reassign
34 | registration.onupdatefound = () => {
35 | const installingWorker = registration.installing;
36 | if (installingWorker == null) {
37 | return;
38 | }
39 | installingWorker.onstatechange = () => {
40 | if (installingWorker.state === 'installed') {
41 | if (navigator.serviceWorker.controller) {
42 | // At this point, the updated precached content has been fetched,
43 | // but the previous service worker will still serve the older
44 | // content until all client tabs are closed.
45 | console.log(
46 | 'New content is available and will be used when all ' +
47 | 'tabs for this page are closed. See https://cra.link/PWA.',
48 | );
49 |
50 | // Execute callback
51 | if (config && config.onUpdate) {
52 | config.onUpdate(registration);
53 | }
54 | } else {
55 | // At this point, everything has been precached.
56 | // It's the perfect time to display a
57 | // "Content is cached for offline use." message.
58 | console.log('Content is cached for offline use.');
59 |
60 | // Execute callback
61 | if (config && config.onSuccess) {
62 | config.onSuccess(registration);
63 | }
64 | }
65 | }
66 | };
67 | };
68 | })
69 | .catch((error) => {
70 | console.error('Error during service worker registration:', error);
71 | });
72 | }
73 |
74 | function checkValidServiceWorker(swUrl: string, config?: Config) {
75 | // Check if the service worker can be found. If it can't reload the page.
76 | fetch(swUrl, {
77 | headers: { 'Service-Worker': 'script' },
78 | })
79 | .then((response) => {
80 | // Ensure service worker exists, and that we really are getting a JS file.
81 | const contentType = response.headers.get('content-type');
82 | if (
83 | response.status === 404 ||
84 | (contentType != null && contentType.indexOf('javascript') === -1)
85 | ) {
86 | // No service worker found. Probably a different app. Reload the page.
87 | void navigator.serviceWorker.ready.then((registration) => {
88 | void registration.unregister().then(() => {
89 | window.location.reload();
90 | });
91 | });
92 | } else {
93 | // Service worker found. Proceed as normal.
94 | registerValidSW(swUrl, config);
95 | }
96 | })
97 | .catch(() => {
98 | console.log(
99 | 'No internet connection found. App is running in offline mode.',
100 | );
101 | });
102 | }
103 |
104 | export function register(config?: Config) {
105 | if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) {
106 | window.addEventListener('load', () => {
107 | const swUrl = '/service-worker.js';
108 |
109 | if (isLocalhost) {
110 | // This is running on localhost. Let's check if a service worker still exists or not.
111 | checkValidServiceWorker(swUrl, config);
112 |
113 | // Add some additional logging to localhost, pointing developers to the
114 | // service worker/PWA documentation.
115 | void navigator.serviceWorker.ready.then(() => {
116 | console.log(
117 | 'This web app is being served cache-first by a service ' +
118 | 'worker. To learn more, visit https://cra.link/PWA',
119 | );
120 | });
121 | } else {
122 | // Is not localhost. Just register service worker
123 | registerValidSW(swUrl, config);
124 | }
125 | });
126 | }
127 | }
128 |
129 | export function unregister() {
130 | if ('serviceWorker' in navigator) {
131 | navigator.serviceWorker.ready
132 | .then((registration) => {
133 | void registration.unregister();
134 | })
135 | .catch((error: Error) => {
136 | console.error(error.message);
137 | });
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/packages/app/src/components/ui/UI.tsx:
--------------------------------------------------------------------------------
1 | import { Control } from './Control';
2 | import { GroupControl } from './GroupControl';
3 | import { GroupControlElement } from './GroupControlElement';
4 | import {
5 | autoDetectAtom,
6 | autoPlayAtom,
7 | debugAtom,
8 | detectingAtom,
9 | flipLayoutAtom,
10 | formatAtom,
11 | layoutAtom,
12 | videoUrlAtom,
13 | } from 'atoms/controls';
14 | import { useAtom, useAtomValue } from 'jotai';
15 | import { useXRSession } from 'hooks/useXRSession';
16 | import clsx from 'clsx';
17 | import type { DropzoneInputProps } from 'react-dropzone';
18 |
19 | export function UI({ fileInputProps }: { fileInputProps: DropzoneInputProps }) {
20 | const [autoPlay, setAutoPlay] = useAtom(autoPlayAtom);
21 | const [autoDetect, setAutoDetect] = useAtom(autoDetectAtom);
22 |
23 | const detecting = useAtomValue(detectingAtom);
24 |
25 | const [layout, setLayout] = useAtom(layoutAtom);
26 | const [flipLayout, setFlipLayout] = useAtom(flipLayoutAtom);
27 | const [format, setFormat] = useAtom(formatAtom);
28 |
29 | const [debug, setDebug] = useAtom(debugAtom);
30 |
31 | const [, setVideoUrl] = useAtom(videoUrlAtom);
32 |
33 | const [xrSupported, xrSession, requestXrSession] = useXRSession();
34 |
35 | return (
36 |
40 |
{
44 | if (xrSession) {
45 | void xrSession.end();
46 | } else {
47 | requestXrSession();
48 | }
49 | }}
50 | >
51 | {
52 | // eslint-disable-next-line no-nested-ternary
53 | xrSupported
54 | ? xrSession
55 | ? 'Exit VR'
56 | : 'Enter VR'
57 | : 'VR Not Supported'
58 | }
59 |
60 |
61 |
68 |
69 |
83 |
84 |
setAutoPlay(!autoPlay)}>
85 | Autoplay
86 |
87 |
88 |
89 | {detecting && (
90 |
91 | Loading...
92 |
96 |
100 |
101 | )}
102 | setAutoDetect(!autoDetect)}
105 | >
106 | Detect Video Settings
107 |
108 |
109 |
110 |
111 |
112 | setLayout('mono')}
115 | >
116 | Mono
117 |
118 | setLayout('stereoLeftRight')}
121 | >
122 | Left | Right
123 |
124 | setLayout('stereoTopBottom')}
127 | >
128 | Top | Bottom
129 |
130 |
131 |
132 |
139 |
152 |
153 |
154 |
155 |
156 | setFormat('screen')}
159 | >
160 | Screen
161 |
162 | setFormat('180')}
165 | >
166 | 180°
167 |
168 | setFormat('360')}
171 | >
172 | 360°
173 |
174 |
175 |
176 |
setDebug(!debug)}>
177 | Preview
178 |
179 |
180 |
187 |
197 |
198 |
199 | );
200 | }
201 |
--------------------------------------------------------------------------------
/packages/app/src/components/App.tsx:
--------------------------------------------------------------------------------
1 | import { ArrowDownTrayIcon, XMarkIcon } from '@heroicons/react/24/solid';
2 | import { DebugPlayer } from 'components/DebugPlayer';
3 | import { Toaster, toast } from 'react-hot-toast';
4 | import { UI } from './ui/UI';
5 | import { VrPlayer } from 'components/VrPlayer';
6 | import {
7 | autoDetectAtom,
8 | autoPlayAtom,
9 | debugAtom,
10 | detectingAtom,
11 | flipLayoutAtom,
12 | formatAtom,
13 | layoutAtom,
14 | videoUrlAtom,
15 | } from 'atoms/controls';
16 | import { getImageFrames } from 'helper/getImageFrames';
17 | import { transfer, wrap } from 'comlink';
18 | import { useAtom, useSetAtom } from 'jotai';
19 | import { useDropzone } from 'react-dropzone';
20 | import { useEffect, useRef, useState } from 'react';
21 | import { useXRSession } from 'hooks/useXRSession';
22 | import clsx from 'clsx';
23 | import type { VideoRecognitionWorker } from 'worker/videoRecognition.worker';
24 |
25 | const worker = wrap(
26 | new Worker(new URL('worker/videoRecognition.worker', import.meta.url)),
27 | );
28 |
29 | export function App() {
30 | const videoRef = useRef(null);
31 | const canvasRef = useRef(null);
32 | const urlInputRef = useRef(null);
33 |
34 | const [layout, setLayout] = useAtom(layoutAtom);
35 | const [flipLayout] = useAtom(flipLayoutAtom);
36 | const [format, setFormat] = useAtom(formatAtom);
37 |
38 | const [debug, setDebug] = useAtom(debugAtom);
39 |
40 | const [autoPlay] = useAtom(autoPlayAtom);
41 | const [autoDetect] = useAtom(autoDetectAtom);
42 |
43 | const [file, setFile] = useState(null);
44 | const [videoUrl, setVideoUrl] = useAtom(videoUrlAtom);
45 | const [ready, setReady] = useState(false);
46 |
47 | const [, xrSession] = useXRSession();
48 |
49 | const setDetecting = useSetAtom(detectingAtom);
50 |
51 | useEffect(() => {
52 | if (ready && videoRef.current && autoDetect) {
53 | setDetecting(true);
54 |
55 | void (async (video) => {
56 | const frames = await getImageFrames(video);
57 |
58 | const [detectedLayout, detectedFormat] = await worker.recognizeVideo(
59 | transfer(frames, [...frames.map((frame) => frame.data.buffer)]),
60 | );
61 |
62 | if (detectedLayout) setLayout(detectedLayout);
63 | if (detectedFormat) setFormat(detectedFormat);
64 |
65 | setDetecting(false);
66 | })(videoRef.current);
67 | }
68 | }, [autoDetect, ready, setDetecting, setFormat, setLayout]);
69 |
70 | useEffect(() => {
71 | let objectUrl = '';
72 |
73 | if (file) {
74 | objectUrl = URL.createObjectURL(file);
75 | setVideoUrl(objectUrl);
76 | }
77 |
78 | return () => {
79 | URL.revokeObjectURL(objectUrl);
80 | };
81 | }, [file, setVideoUrl]);
82 |
83 | useEffect(() => {
84 | if (ready && urlInputRef.current) {
85 | urlInputRef.current.value = '';
86 | }
87 | }, [ready]);
88 |
89 | useEffect(() => {
90 | if (videoRef.current) {
91 | setReady(false);
92 | videoRef.current.src = videoUrl ?? '';
93 | }
94 | }, [videoUrl]);
95 |
96 | const { getRootProps, getInputProps, isDragActive } = useDropzone({
97 | noClick: true,
98 | multiple: false,
99 | accept: {
100 | 'video/*': [],
101 | },
102 | onDropAccepted: (acceptedFiles) => {
103 | setFile(acceptedFiles[0]);
104 | },
105 | onDropRejected: (rejection) => {
106 | toast.error(rejection[0].errors[0].message);
107 | },
108 | });
109 |
110 | return (
111 |
115 |
116 | {videoRef.current && canvasRef.current && ready && debug && (
117 |
124 | )}
125 | {videoRef.current && canvasRef.current && ready && xrSession && (
126 |
134 | )}
135 |
136 |
137 |
138 |
139 |
197 |
205 |
206 |
216 |
217 |
230 |
231 | );
232 | }
233 |
--------------------------------------------------------------------------------
/org/icon.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
140 |
--------------------------------------------------------------------------------