feat(web): stream video and audio while decrypting, paste images, 16px inputs on touch

This commit is contained in:
justme committed 2026-10-17 14:35:41 +08:00
1 parent 31ae3bc39e
commit 283196938e
12 files changed
+376 -16

No files matched your search

+79
View File
@@ -0,0 +1,79 @@
// 加密视频/音频的边下边播。
// 播放器请求 /__wd_stream/<id>(带 Range),这里不联网:把「要哪一段」告诉发起请求的页面,
// 页面把一个 MessagePort 交给加密线程,加密线程解密出这一段后一块一块从端口推回来,
// 这里再把它们作为 206 响应流式交给播放器。密钥始终只在加密线程里。
const PREFIX = '/__wd_stream/';
/** 一次最多让加密线程领先播放器多少块,避免整个文件都堆在内存里 */
const WINDOW = 6;
self.addEventListener('install', () => self.skipWaiting());
self.addEventListener('activate', (event) => event.waitUntil(self.clients.claim()));
self.addEventListener('fetch', (event) => {
const url = new URL(event.request.url);
if (url.origin !== self.location.origin || !url.pathname.startsWith(PREFIX)) return;
event.respondWith(serve(event).catch((error) => new Response(String(error && error.message || error), { status: 500 })));
});
function parseRange(header) {
const m = /^bytes=(\d*)-(\d*)$/.exec(header || '');
if (!m) return { start: 0, end: undefined };
if (m[1] === '' && m[2] !== '') return { suffix: Number(m[2]) };
return { start: Number(m[1] || 0), end: m[2] === '' ? undefined : Number(m[2]) };
}
async function serve(event) {
const id = decodeURIComponent(new URL(event.request.url).pathname.slice(PREFIX.length));
const client = event.clientId ? await self.clients.get(event.clientId) : undefined;
const target = client ?? (await self.clients.matchAll({ type: 'window' }))[0];
if (!target) return new Response('no client', { status: 503 });
const range = parseRange(event.request.headers.get('Range'));
const channel = new MessageChannel();
const port = channel.port1;
// 第一条消息:{ total, mime, start, end } 或 { error };之后是 { data } / { done } / { error }
const first = new Promise((resolve) => {
port.onmessage = (message) => resolve(message.data);
});
target.postMessage({ type: 'wd-stream', id, range, port: channel.port2 }, [channel.port2]);
const head = await Promise.race([first, new Promise((resolve) => setTimeout(() => resolve({ error: 'timeout' }), 20000))]);
if (head.error) return new Response(head.error, { status: head.status || 502 });
const { total, mime, start, end } = head;
let pending = [];
let waiting;
let finished = false;
let failure;
port.onmessage = (message) => {
const m = message.data;
if (m.data) pending.push(m.data);
else if (m.done) finished = true;
else if (m.error) failure = m.error;
if (waiting) { const wake = waiting; waiting = undefined; wake(); }
};
port.postMessage({ credit: WINDOW });
const body = new ReadableStream({
async pull(controller) {
while (!pending.length && !finished && !failure) await new Promise((resolve) => { waiting = resolve; });
if (pending.length) {
controller.enqueue(new Uint8Array(pending.shift()));
port.postMessage({ credit: 1 });
} else if (failure) { controller.error(new Error(failure)); port.close(); }
else { controller.close(); port.close(); }
},
cancel() { port.postMessage({ cancel: true }); port.close(); },
});
const partial = !(start === 0 && end === total - 1 && !event.request.headers.get('Range'));
return new Response(body, {
status: partial ? 206 : 200,
headers: {
'Content-Type': mime || 'application/octet-stream',
'Content-Length': String(end - start + 1),
'Accept-Ranges': 'bytes',
'Cache-Control': 'no-store',
...(partial ? { 'Content-Range': `bytes ${start}-${end}/${total}` } : {}),
},
});
}
+48
View File
@@ -0,0 +1,48 @@
import { postToWorker } from './workerClient';
/**
* 页面这一侧的「边下边播」:把条目登记成一个 /__wd_stream/<id> 地址交给 <video>/<audio>,
* 播放器的每个 Range 请求由 Service Worker(public/stream-sw.js)转给这里,
* 这里把 SW 给的 MessagePort 直接交给加密线程,解密后的数据不再经过页面。
*/
interface Entry { resourceId: string; itemId: string; size: number; mime: string }
interface SwMessage { type: 'wd-stream'; id: string; range: { start?: number; end?: number; suffix?: number }; port: MessagePort }
const entries = new Map<string, Entry>();
let ready: Promise<boolean> | undefined;
function setup(): Promise<boolean> {
if (!('serviceWorker' in navigator)) return Promise.resolve(false);
return (async () => {
navigator.serviceWorker.addEventListener('message', (event: MessageEvent<SwMessage>) => {
const message = event.data;
if (message?.type !== 'wd-stream') return;
const entry = entries.get(message.id);
if (!entry) { message.port.postMessage({ error: 'gone', status: 404 }); message.port.close(); return; }
postToWorker({ operation: 'streamRange', payload: { ...entry, ...message.range } }, [message.port]);
});
await navigator.serviceWorker.register(`${import.meta.env.BASE_URL}stream-sw.js`);
await navigator.serviceWorker.ready;
if (!navigator.serviceWorker.controller) {
// 第一次安装:等 SW 接管当前页面(clients.claim)
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, 2000);
navigator.serviceWorker.addEventListener('controllerchange', () => { clearTimeout(timer); resolve(); }, { once: true });
});
}
return navigator.serviceWorker.controller !== null;
})().catch(() => false);
}
/** 登记一个可以边下边播的条目,返回播放地址;浏览器或环境不支持时返回 undefined(调用方退回整个下载) */
export async function playbackUrl(resourceId: string, itemId: string, size: number, mime: string): Promise<string | undefined> {
ready ??= setup();
if (!(await ready)) return undefined;
const id = `${crypto.randomUUID()}`;
entries.set(id, { resourceId, itemId, size, mime });
return `${import.meta.env.BASE_URL}__wd_stream/${id}`;
}
export function releasePlayback(url: string): void {
entries.delete(url.slice(url.lastIndexOf('/') + 1));
}
@@ -0,0 +1,125 @@
import { fromBase64Url, IntegrityError } from '@webdrop/core';
import { apiFetch } from './http';
import { getSession } from './sessions';
/**
* 边下边播:按字节范围把加密条目解密后推给 Service Worker(stream-sw.js)。
*
* 条目的帧是定长的:第 0 帧内容 cap0 字节,之后每帧 maxData 字节,最后一帧不足。
* 读完第 0、1 帧就知道这两个数,于是任意字节位置都能算出在第几帧,从那一帧开始拉取(from 参数),
* 拖进度条不用从头下载。每一帧都有 AEAD 认证(绑定空间、条目和帧序号),被篡改的帧直接报错;
* 整个条目的清单校验需要读完全部帧,边播不做(下载保存时仍然会做)。
*/
interface Geometry { cap0: number; maxData: number; head: Uint8Array[] }
const geometries = new Map<string, Promise<Geometry>>();
interface PlaybackRequest { resourceId: string; itemId: string; size: number; mime: string; start?: number; end?: number; suffix?: number }
async function framesFrom(resourceId: string, itemId: string, from: number): Promise<{ next: () => Promise<Uint8Array | undefined>; close: () => Promise<void> }> {
const { session, resource } = await getSession(resourceId);
if (resource.kind !== 'channel') throw new Error('资源类型不匹配');
const role = session.can('read') ? 'read' : 'owner';
const response = await apiFetch(`/channels/${resourceId}/items/${itemId}/frames?from=${from}`, { headers: session.authHeaders(role) });
if (!response.body) throw new Error('服务端没有返回文件内容');
const expectedId = fromBase64Url(itemId, 16);
if (!expectedId) throw new IntegrityError('bad_meta');
// 每一帧解出来就是一块内容,块的边界就是帧的边界
const { content } = session.decryptItemStream(response.body, { itemId: expectedId, fromIndex: from });
const reader = content.getReader();
return {
next: async () => { const { done, value } = await reader.read(); return done ? undefined : value; },
close: () => reader.cancel().catch(() => undefined),
};
}
function geometryOf(resourceId: string, itemId: string, size: number): Promise<Geometry> {
const key = `${resourceId}:${itemId}`;
let known = geometries.get(key);
if (!known) {
if (geometries.size >= 8) geometries.clear();
known = (async () => {
const frames = await framesFrom(resourceId, itemId, 0);
try {
const first = await frames.next();
if (!first) throw new Error('文件内容为空');
if (first.length >= size) return { cap0: first.length, maxData: Math.max(1, first.length), head: [first] };
const second = await frames.next();
if (!second) throw new Error('文件不完整');
return { cap0: first.length, maxData: Math.max(1, second.length), head: [first, second] };
} finally { await frames.close(); }
})();
known.catch(() => geometries.delete(key));
geometries.set(key, known);
}
return known;
}
/** 字节位置 → 所在的帧序号和帧内偏移 */
function locate(g: Geometry, position: number): { index: number; offset: number } {
if (position < g.cap0) return { index: 0, offset: position };
const rest = position - g.cap0;
return { index: 1 + Math.floor(rest / g.maxData), offset: rest % g.maxData };
}
/** 流控:对方(Service Worker)每消费一块才给一个额度,避免把整个文件一口气解密进内存 */
function flow(port: MessagePort) {
let credit = 0; let cancelled = false; let wake: (() => void) | undefined;
port.onmessage = (message: MessageEvent<{ credit?: number; cancel?: boolean }>) => {
if (message.data.cancel) cancelled = true;
if (message.data.credit) credit += message.data.credit;
wake?.(); wake = undefined;
};
return {
get cancelled() { return cancelled; },
async take(): Promise<boolean> {
while (credit <= 0 && !cancelled) await new Promise<void>((resolve) => { wake = resolve; });
if (cancelled) return false;
credit -= 1; return true;
},
};
}
export async function servePlaybackRange(port: MessagePort, request: PlaybackRequest): Promise<void> {
const { resourceId, itemId, size, mime } = request;
const control = flow(port);
try {
if (request.suffix !== undefined && request.suffix <= 0) { port.postMessage({ error: 'bad range', status: 416 }); return; }
const start = request.suffix !== undefined ? Math.max(0, size - request.suffix) : request.start ?? 0;
const end = Math.min(size - 1, request.end ?? size - 1);
if (start >= size || start > end) { port.postMessage({ error: 'range not satisfiable', status: 416 }); return; }
const geometry = await geometryOf(resourceId, itemId, size);
port.postMessage({ total: size, mime, start, end });
let position = start;
const send = async (chunk: Uint8Array, offset: number): Promise<boolean> => {
if (!(await control.take())) return false;
const part = chunk.slice(offset, offset + (end + 1 - position));
const length = part.length; // 转移之后 part 就被分离了,长度会变成 0
port.postMessage({ data: part.buffer }, [part.buffer]);
position += length;
return true;
};
let { index, offset } = locate(geometry, position);
// 第 0、1 帧已经在手上
while (position <= end && index < geometry.head.length) {
if (!(await send(geometry.head[index]!, offset))) return;
index += 1; offset = 0;
}
if (position <= end) {
const frames = await framesFrom(resourceId, itemId, index);
try {
while (position <= end) {
const chunk = await frames.next();
if (!chunk) throw new Error('文件不完整');
if (!(await send(chunk, offset))) return;
offset = 0;
}
} finally { await frames.close(); }
}
port.postMessage({ done: true });
} catch (error) {
port.postMessage({ error: error instanceof Error ? error.message : '播放失败' });
}
// 端口由 Service Worker 读完后关闭:这里立刻关掉会让还没送达的最后几条消息丢失
}
+5 -1
View File
@@ -17,7 +17,11 @@ let clientHintsPromise: Promise<ClientHints> | undefined;
async function readClientHints(): Promise<ClientHints> {
const hints = (navigator as Navigator & { userAgentData?: ClientHints }).userAgentData;
if (!hints?.getHighEntropyValues) return hints ?? {};
clientHintsPromise ??= hints.getHighEntropyValues(['model', 'platform', 'platformVersion']).catch(() => hints);
// 个别手机浏览器的 getHighEntropyValues 会很久没有响应,不能因此卡住上传:最多等 1.5 秒,超时就用基础信息
clientHintsPromise ??= Promise.race([
hints.getHighEntropyValues(['model', 'platform', 'platformVersion']).catch(() => hints),
new Promise<ClientHints>((resolve) => setTimeout(() => resolve(hints), 1500)),
]);
return clientHintsPromise;
}
+7 -6
View File
@@ -4,6 +4,7 @@ import { formatSize } from '@/lib/format';
import { Icon } from '@/ui/Icon';
import { ArchivePreview } from '../item/previews/ArchivePreview';
import { useBlobUrl } from '../item/useBlobUrl';
import { useMediaUrl } from '../item/useMediaUrl';
import styles from './ChatAttachment.module.css';
/** 聊天里自动展开的大小上限:再大的只显示文件卡片,点开预览 */
@@ -54,15 +55,15 @@ function Picture({ item, onOpen }: { item: FileItem; onOpen: () => void }) {
}
function Clip({ item }: { item: FileItem }) {
const blob = useBlobUrl(item.id);
if (blob.status === 'loading') return <div className={styles.placeholder}>加载中…</div>;
if (blob.status === 'error') return null;
return <video className={styles.video} src={blob.url} controls preload="metadata" playsInline />;
const { state, onError } = useMediaUrl(item.id);
if (state.status === 'loading') return <div className={styles.placeholder}>加载中…</div>;
if (state.status === 'error') return null;
return <video className={styles.video} src={state.url} controls preload="metadata" playsInline onError={onError} />;
}
function Sound({ item }: { item: FileItem }) {
const blob = useBlobUrl(item.id);
return blob.status === 'ready' ? <audio className={styles.audio} src={blob.url} controls /> : null;
const { state, onError } = useMediaUrl(item.id);
return state.status === 'ready' ? <audio className={styles.audio} src={state.url} controls preload="metadata" onError={onError} /> : null;
}
const LINES = 12;
@@ -274,9 +274,17 @@
color: var(--text);
font: inherit;
font-size: 14px;
font-weight: 400;
letter-spacing: normal;
line-height: 20px;
}
.composer textarea::placeholder {
color: var(--text-dim);
font-weight: 400;
opacity: 1;
}
@media (width <= 899px) {
/* 手机顶栏已显示频道名 */
.head {
@@ -301,6 +309,12 @@
.composer {
margin: var(--space-2);
}
/* iOS 的输入框字号小于 16px 时,点进去页面会被放大,布局跟着错位 */
.composer textarea {
font-size: 16px;
line-height: 22px;
}
}
/* 触屏没有悬停:隐藏常驻按钮,点一下消息弹出菜单 */
@@ -332,3 +346,13 @@
flex: 1;
white-space: nowrap;
}
.avatarButton {
display: block;
flex: none;
padding: 0;
border: 0;
border-radius: var(--radius-avatar);
background: none;
cursor: zoom-in;
}
+17 -2
View File
@@ -1,4 +1,4 @@
import { Fragment, useEffect, useRef, useState, type KeyboardEvent } from 'react';
import { Fragment, useEffect, useRef, useState, type ClipboardEvent, type KeyboardEvent } from 'react';
import { Button } from '@/ui/Button';
import { useWorkspace } from '@/store/workspace';
import type { Item } from '@/core-bridge/types';
@@ -36,6 +36,21 @@ export function ChatPane({ threadName, ids, enabled, onSend, onDelete, onOpenFil
void onSend(value);
}
// 粘贴图片(截图、复制的图片)直接当文件发送;只有文字的粘贴照常进输入框
function paste(event: ClipboardEvent<HTMLTextAreaElement>) {
if (!onFiles || !enabled) return;
const images = [...event.clipboardData.files].filter((file) => file.type.startsWith('image/'));
if (!images.length) return;
event.preventDefault();
const stamp = new Date().toLocaleString('sv').replace(/[: ]/g, '-');
onFiles(images.map((file, index) => {
// 截图的文件名通常就是 image.png,换成带时间的名字,不然连续发几张分不清
const generic = /^image\.\w+$/i.test(file.name) || !file.name;
const extension = file.type.split('/')[1]?.replace('jpeg', 'jpg') ?? 'png';
return generic ? new File([file], `粘贴的图片 ${stamp}${images.length > 1 ? `-${index + 1}` : ''}.${extension}`, { type: file.type, lastModified: file.lastModified }) : file;
}));
}
function keyDown(event: KeyboardEvent<HTMLTextAreaElement>) {
if (event.key === 'Enter' && !event.shiftKey && !event.nativeEvent.isComposing) { event.preventDefault(); submit(); }
}
@@ -67,7 +82,7 @@ export function ChatPane({ threadName, ids, enabled, onSend, onDelete, onOpenFil
{onFiles ? <><input ref={fileInput} type="file" multiple hidden onChange={(event) => { if (event.target.files?.length) onFiles([...event.target.files]); event.target.value = ''; }} />
<Button variant="ghost" icon="plus" label="上传文件到当前频道" disabled={!enabled} onClick={() => fileInput.current?.click()} /></> : null}
<textarea ref={input} value={text} disabled={!enabled} rows={1} aria-label="消息内容" placeholder={enabled ? `给 #${threadName} 发消息` : '先创建或加入一个频道'}
onChange={(event) => { setText(event.target.value); fit(); }} onKeyDown={keyDown} />
onChange={(event) => { setText(event.target.value); fit(); }} onKeyDown={keyDown} onPaste={paste} />
<Button type="submit" variant="ghost" icon="send" label="发送消息" disabled={!enabled || !text.trim()} />
</form>
</section>
@@ -1,5 +1,6 @@
import { useState } from 'react';
import { useBlobUrl } from '../useBlobUrl';
import { useMediaUrl } from '../useMediaUrl';
import styles from './previews.module.css';
interface Props {
@@ -7,15 +8,27 @@ interface Props {
kind: 'image' | 'video' | 'audio';
}
/** 媒体用原生元素加载 blob URL,不执行任何内容里的脚本 */
/** 媒体用原生元素加载:视频、音频边下边播,图片取整个内容的 blob URL;都不执行内容里的脚本 */
export function MediaPreview({ id, kind }: Props) {
if (kind === 'image') return <ImagePreview id={id} />;
return <PlayerPreview id={id} kind={kind} />;
}
function PlayerPreview({ id, kind }: { id: string; kind: 'video' | 'audio' }) {
const { state, onError } = useMediaUrl(id);
if (state.status === 'loading') return <p className={styles.hint}>加载中…</p>;
if (state.status === 'error') return <p className={styles.hint}>无法预览此内容,可以直接下载。</p>;
return kind === 'video'
? <video className={styles.media} src={state.url} controls playsInline onError={onError} />
: <audio className={styles.audio} src={state.url} controls onError={onError} />;
}
function ImagePreview({ id }: { id: string }) {
const blob = useBlobUrl(id);
const [broken, setBroken] = useState(false);
if (blob.status === 'loading') return <p className={styles.hint}>加载中…</p>;
if (blob.status === 'error') return <p className={styles.hint}>无法预览此内容,可以直接下载。</p>;
if (broken) return <p className={styles.hint}>此浏览器无法显示该格式(例如 iPhone 的 HEIC 照片),可以直接下载。</p>;
if (kind === 'image') return <img className={styles.media} src={blob.url} alt="" onError={() => setBroken(true)} />;
if (kind === 'video') return <video className={styles.media} src={blob.url} controls playsInline onError={() => setBroken(true)} />;
return <audio className={styles.audio} src={blob.url} controls />;
return <img className={styles.media} src={blob.url} alt="" onError={() => setBroken(true)} />;
}
+9
View File
@@ -1,6 +1,7 @@
import { createContext, useContext } from 'react';
import type { ArchiveEntry, Item } from '@/core-bridge/types';
import { UNKNOWN_SENDER_LABEL } from '@/lib/sender';
import { playbackUrl } from '@/core-bridge/streamClient';
import { useShareSession } from '@/store/shareSession';
import { useWorkspace } from '@/store/workspace';
@@ -20,6 +21,8 @@ export interface ItemSource {
fallbackSender: string;
getBlob: (id: string, versionId?: string) => Promise<Blob>;
listArchive: (id: string) => Promise<ArchiveEntry[]>;
/** 视频、音频的边下边播地址;分享不支持,浏览器不支持时也返回 undefined(退回整个下载) */
streamUrl?: (id: string) => Promise<string | undefined>;
/** 只有空间有:分享是只读的 */
actions?: ItemActions;
}
@@ -29,6 +32,12 @@ export const channelSource: ItemSource = {
fallbackSender: UNKNOWN_SENDER_LABEL,
getBlob: (id, versionId) => useWorkspace.getState().getBlob(id, versionId),
listArchive: (id) => useWorkspace.getState().listArchive(id),
streamUrl: async (id) => {
const { channel, byId } = useWorkspace.getState();
const item = byId[id];
if (!channel || item?.kind !== 'file') return undefined;
return playbackUrl(channel.id, id, item.size, item.mime);
},
actions: {
markSeen: (id) => useWorkspace.getState().markSeen(id),
editText: (id, text) => useWorkspace.getState().editText(id, text),
+20
View File
@@ -0,0 +1,20 @@
import { useState } from 'react';
import { useBlobUrl } from './useBlobUrl';
import { useStreamUrl } from './useStreamUrl';
type MediaState = { status: 'loading' } | { status: 'ready'; url: string } | { status: 'error' };
/**
* 视频、音频的播放地址:优先边下边播(只拉播放器要的那一段,拖进度条也不用从头下载);
* 浏览器不支持或播放出错时,退回整个下载完再播放。
*/
export function useMediaUrl(id: string): { state: MediaState; onError: () => void } {
const stream = useStreamUrl(id);
const [failed, setFailed] = useState(false);
const fallback = stream.status === 'unavailable' || failed;
const blob = useBlobUrl(id, fallback);
const state: MediaState = fallback
? blob.status === 'ready' ? { status: 'ready', url: blob.url } : blob
: stream.status === 'ready' ? { status: 'ready', url: stream.url } : { status: 'loading' };
return { state, onError: () => setFailed(true) };
}
+22
View File
@@ -0,0 +1,22 @@
import { useEffect, useState } from 'react';
import { releasePlayback } from '@/core-bridge/streamClient';
import { useItemSource } from './source';
type StreamState = { status: 'loading' } | { status: 'ready'; url: string } | { status: 'unavailable' };
/** 视频、音频边下边播的地址。不支持时返回 unavailable,调用方退回整个下载再播放 */
export function useStreamUrl(id: string, enabled = true): StreamState {
const { streamUrl } = useItemSource();
const [state, setState] = useState<StreamState>({ status: 'loading' });
useEffect(() => {
if (!enabled) return;
if (!streamUrl) { setState({ status: 'unavailable' }); return; }
let url: string | undefined; let cancelled = false;
void streamUrl(id).then((value) => {
if (cancelled) { if (value) releasePlayback(value); return; }
url = value; setState(value ? { status: 'ready', url: value } : { status: 'unavailable' });
}).catch(() => { if (!cancelled) setState({ status: 'unavailable' }); });
return () => { cancelled = true; if (url) releasePlayback(url); };
}, [id, enabled, streamUrl]);
return state;
}
+3 -3
View File
@@ -43,12 +43,12 @@ summary,
touch-action: manipulation;
}
/* iOS Safari 对小于 16px 的输入框聚焦时会自动放大页面 */
/* iOS Safari 对小于 16px 的输入框聚焦时会自动放大页面。各组件自己的样式(如登录页的 14px)优先级更高,所以这里用 !important 兜底 */
@media (pointer: coarse) {
input:not([type="checkbox"], [type="radio"]),
input:not([type="checkbox"], [type="radio"], [type="range"], [type="color"]),
textarea,
select {
font-size: 16px;
font-size: 16px !important;
}
}