From 23f3a15a15a5b6e697eabd62092ab99cfc9d4066 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 14:00:14 +0800 Subject: [PATCH] feat: add browser workbench and managed local launchers --- frontend/index.html | 2 + frontend/src/App.tsx | 128 +++++++++++++++++++ frontend/src/api.ts | 9 ++ frontend/src/format.test.ts | 17 +++ frontend/src/format.ts | 16 +++ frontend/src/main.tsx | 5 + frontend/src/style.css | 243 ++++++++++++++++++++++++++++++++++++ frontend/src/types.ts | 12 ++ frontend/src/vite-env.d.ts | 1 + requirements.lock | 34 +++++ start.ps1 | 56 +++++++++ stop.ps1 | 15 +++ studio/connectors.py | 1 + tests/test_collectors.py | 7 ++ tools/browser_qa.py | 75 +++++++++++ 停止工作台.cmd | 3 + 启动工作台.cmd | 3 + 17 files changed, 627 insertions(+) create mode 100644 frontend/index.html create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/api.ts create mode 100644 frontend/src/format.test.ts create mode 100644 frontend/src/format.ts create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/style.css create mode 100644 frontend/src/types.ts create mode 100644 frontend/src/vite-env.d.ts create mode 100644 requirements.lock create mode 100644 start.ps1 create mode 100644 stop.ps1 create mode 100644 tools/browser_qa.py create mode 100644 停止工作台.cmd create mode 100644 启动工作台.cmd diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..552efd2 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,2 @@ + +研途 · 考研数据工作台
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..736a5d8 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,128 @@ +import { useState, useEffect, useCallback, useId } from 'react'; +import type { ReactNode } from 'react'; +import { ArrowRight, ArrowUpRight, BarChart3, BookOpen, Check, ChevronLeft, ChevronRight, Database, Download, FileUp, Globe2, GraduationCap, LayoutDashboard, ListFilter, Loader2, Network, Pause, Play, Plus, RefreshCw, Search, Settings2, ShieldCheck, Square, Star, X } from 'lucide-react'; +import { api, post } from './api'; +import { display, scopeName, statusName, modeName, queryString, scoreSeries } from './format'; +import type { Row, Overview, Province, Major, Job, Log, Config, Proxy, Auth, History, Filters } from './types'; + +type Tab = 'overview' | 'workspace' | 'jobs' | 'proxy' | 'settings'; +type Notify = (message: string, error?: boolean) => void; +const NAV = [{ key: 'overview', label: '数据概览', icon: LayoutDashboard }, { key: 'workspace', label: '院校工作区', icon: GraduationCap }, { key: 'jobs', label: '采集任务', icon: RefreshCw }, { key: 'proxy', label: '代理 IP 池', icon: Network }, { key: 'settings', label: '设置与登录', icon: Settings2 }] as const; +const currentYear = new Date().getFullYear(); +const activeStates = ['queued', 'running']; +const resumeStates = ['paused', 'partial', 'failed', 'waiting_login', 'waiting_proxy']; + +function Badge({ children, tone = '' }: { children: ReactNode; tone?: string }) { return {children}; } +function Empty({ title, children }: { title: string; children?: ReactNode }) { return

{title}

{children}
; } +function Modal({ title, children, close, wide = false }: { title: string; children: ReactNode; close: () => void; wide?: boolean }) { + useEffect(() => { const listener = (e: KeyboardEvent) => { if (e.key === 'Escape') close(); }; document.addEventListener('keydown', listener); const old = document.body.style.overflow; document.body.style.overflow = 'hidden'; return () => { document.removeEventListener('keydown', listener); document.body.style.overflow = old; }; }, [close]); + return
e.stopPropagation()}>

{title}

{children}
; +} +function MajorInput({ majors, value, onChange, label = '专业代码', required = false }: { majors: Major[]; value: string; onChange: (value: string) => void; label?: string; required?: boolean }) { + const id = useId(); + return ; +} +function Stat({ label, value, icon, sub }: { label: string; value: number; icon: ReactNode; sub: string }) { return
{label}{icon}
{Number(value || 0).toLocaleString()}{sub}
; } + +export default function App() { + const [tab, setTab] = useState('overview'); + const [overview, setOverview] = useState(null); + const [majors, setMajors] = useState([]); + const [version, setVersion] = useState(0); + const [notice, setNotice] = useState<{ message: string; error: boolean } | null>(null); + const notify = useCallback((message, error = false) => setNotice({ message, error }), []); + const refresh = useCallback(() => setVersion(v => v + 1), []); + useEffect(() => { const controller = new AbortController(); Promise.all([api('/overview', { signal: controller.signal }), api('/majors', { signal: controller.signal })]).then(([o, m]) => { setOverview(o); setMajors(m); }).catch(e => { if (e.name !== 'AbortError') notify(e.message, true); }); return () => controller.abort(); }, [version, notify]); + useEffect(() => { if (!notice) return; const timer = setTimeout(() => setNotice(null), 6500); return () => clearTimeout(timer); }, [notice]); + const props = { overview, majors, notify, refresh }; + return
工作台 {NAV.find(n => n.key === tab)?.label}
本地运行
{tab === 'overview' && }{tab === 'workspace' && }{tab === 'jobs' && }{tab === 'proxy' && }{tab === 'settings' && }
研途 · 数据以院校当年招生简章为准全国招生目录 / 历年考情 / 本地工作区
{notice &&
{notice.error ? : }{notice.message}
}
; +} + +type Shared = { overview: Overview | null; majors: Major[]; notify: Notify; refresh: () => void }; +function PageTitle({ eyebrow, title, description, children }: { eyebrow: string; title: string; description: string; children?: ReactNode }) { return
{eyebrow}

{title}

{description}

{children}
; } + +function Dashboard({ overview: o, go }: Shared & { go: (tab: Tab) => void }) { + const [jobs, setJobs] = useState([]); + useEffect(() => { const controller = new AbortController(); api('/jobs', { signal: controller.signal }).then(setJobs).catch(() => {}); return () => controller.abort(); }, []); + return <>
} sub={`另有 ${o?.indexed_schools || 0} 所院校公开索引`} />} sub="按年份、方向和科目组合保存" />} sub="保留专业、学科与门类口径" />} sub="在工作区筛选与横向对比" />
开始你的研究

一个工作区,两种数据来源

先建立招生目录,再补充历年分数线和计划。已有数据也可以直接导入,继续筛选、收藏与对比。

01
连接账号与代理池登录学信网,选择当前网络策略
02
选择年份、地区和专业按省采集,也可覆盖全国 31 个省级地区
03
把数据变成择校清单多条件筛选、历史趋势和 Excel 导出

已接入数据来源

2 个来源
{o?.sources.map((s, i) => {i === 0 ? : }
{s.name}{s.role}
)}

数据保存在本机。每条记录都保留来源和年份,未知字段明确标记。

最近采集任务

{jobs.length ?
{jobs.slice(0, 4).map(j =>
{modeName(j.params.mode)} · {j.params.year} 年{j.message}
{statusName(j.status)}{j.completed_units} 个完成单元
)}
:
还没有采集任务

创建一个任务,或在院校工作区导入 YAM / yzw 的已有数据。

}
; +} + +function Workspace({ overview, majors, notify, refresh }: Shared) { + const [filters, setFilters] = useState({ sort: 'year' }); + const [page, setPage] = useState(1); + const [rows, setRows] = useState([]); + const [total, setTotal] = useState(0); + const [loading, setLoading] = useState(true); + const [localVersion, setLocalVersion] = useState(0); + const [selected, setSelected] = useState([]); + const [compare, setCompare] = useState(false); + const [detail, setDetail] = useState(null); + const [importing, setImporting] = useState(false); + const [more, setMore] = useState(false); + const query = queryString({ ...filters, page, page_size: 30 }); + useEffect(() => { const controller = new AbortController(); setLoading(true); const timer = setTimeout(() => api<{ items: Row[]; total: number }>('/catalog?' + query, { signal: controller.signal }).then(data => { setRows(data.items); setTotal(data.total); }).catch(e => { if (e.name !== 'AbortError') notify(e.message, true); }).finally(() => { if (!controller.signal.aborted) setLoading(false); }), 180); return () => { clearTimeout(timer); controller.abort(); }; }, [query, localVersion, notify]); + const change = (key: string, value: string | boolean) => { setFilters(f => ({ ...f, [key]: value })); setPage(1); }; + async function favorite(row: Row) { try { await api(`/catalog/${row.id}/favorite`, { method: 'PUT', body: JSON.stringify({ enabled: !row.favorite }) }); setLocalVersion(v => v + 1); refresh(); } catch (e) { notify((e as Error).message, true); } } + const toggleSelected = (row: Row) => { if (selected.some(s => s.id === row.id)) setSelected(s => s.filter(v => v.id !== row.id)); else if (selected.length < 6) setSelected(s => [...s, row]); else notify('最多同时对比 6 条记录', true); }; + return <>
{more &&
change('major_code', v)} />
}
{total.toLocaleString()} 条记录 · 人数保留原文,科目组合分别展示{loading && }
{rows.length ?
{rows.map(row => )}
对比院校 / 专业院系 / 方向初试科目招生人数最近参考线年份 / 来源收藏
s.id === row.id)} onChange={() => toggleSelected(row)} />{row.major_code} · {display(row.major_name)}
{row.province && {row.province}}{row.level && {row.level}}{row.study_mode && {row.study_mode}}
{display(row.department_name)}
{display(row.direction)}
{row.subjects.some(Boolean) ? row.subjects.map((s, i) => s &&
{s}
) : 未提供}
{display(row.quota)}{row.quota_text || '来源未提供人数'}{row.score_total !== null ? <>{row.score_total}{row.score_year} · {scopeName(row.score_scope)} : 未提供}{row.year}{row.source}{!row.year_verified && 年份由文件指定}
: !loading ?

可以调整筛选条件,或导入 YAM / yzw 数据、创建新的采集任务。

:
正在读取本地数据…
}
第 {page} / {Math.max(1, Math.ceil(total / 30))} 页 · 每页 30 条
{selected.length > 0 &&
已选 {selected.length} 条记录{selected.map(s => s.school_name).join(' / ')}
}{detail !== null && setDetail(null)} notify={notify} />}{compare && setCompare(false)} />}{importing && setImporting(false)} notify={notify} done={() => { setLocalVersion(v => v + 1); refresh(); }} />}; +} + +function ImportDialog({ close, done, notify }: { close: () => void; done: () => void; notify: Notify }) { + const [file, setFile] = useState(null), [year, setYear] = useState(String(currentYear)), [busy, setBusy] = useState(false); + async function submit(e: React.FormEvent) { e.preventDefault(); if (!file) return; const body = new FormData(); body.set('file', file); if (year) body.set('year', year); setBusy(true); try { const result = await api<{ records: number; scores: number }>('/import', { method: 'POST', body }); notify(`已导入 ${result.records} 条目录/计划、${result.scores} 条分数线`); done(); close(); } catch (e) { notify((e as Error).message, true); } finally { setBusy(false); } } + return

支持 yzw CSV、工作台 JSON / Excel,以及 YAM SQLite 数据库。相同记录按年份更新,其他年份保留。

文件本身有年份时使用文件年份。yzw CSV 和 YAM 无年份目录必须明确填写;分数线仍保留其原始年份。
; +} + +function Comparison({ rows, close }: { rows: Row[]; close: () => void }) { + const fields: [string, (row: Row) => ReactNode][] = [['专业', r => `${r.major_code} ${r.major_name}`], ['地区 / 层次', r => `${r.province} ${r.level}`], ['院系', r => display(r.department_name)], ['方向', r => display(r.direction)], ['学制形式', r => display(r.study_mode)], ['初试科目', r => r.subjects.filter(Boolean).map((s, i) =>
{s}
)], ['人数原文', r => display(r.quota_text)], ['明确人数 / 口径', r => `${display(r.quota)} / ${r.quota_scope === 'major' ? '专业' : r.quota_scope === 'college' ? '学院' : '口径待核实'}`], ['最近参考线', r => r.score_total !== null ? `${r.score_year} 年 ${r.score_total} · ${scopeName(r.score_scope)}` : '未提供'], ['导师', r => display(r.advisor)], ['专项计划', r => r.special_plans.join(' / ') || '未提供'], ['来源', r => `${r.year} 年 · ${r.source}`]]; + return

每列是一条独立方向/科目组合。专业人数可能重复展示,不能按列相加。

{rows.map(r => )}{fields.map(([label, field]) => {rows.map(r => )})}
比较项{r.school_name}{r.year} 年
{label}{field(r)}
; +} + +function Detail({ rid, close, notify }: { rid: number; close: () => void; notify: Notify }) { + const [data, setData] = useState(null); + useEffect(() => { const controller = new AbortController(); api(`/catalog/${rid}`, { signal: controller.signal }).then(setData).catch(e => { if (e.name !== 'AbortError') notify(e.message, true); }); return () => controller.abort(); }, [rid, notify]); + const r = data?.record; + return {data && r ?
{r.year} 年{r.province || '地区未提供'}{r.source}{r.year_verified ? '来源年份已核实' : '年份由导入文件指定'}{/^https:\/\//.test(r.source_url) && 查看来源 }
{[['院系', r.department_name], ['研究方向', r.direction], ['学习方式', r.study_mode], ['学位类型', r.degree_type], ['导师', r.advisor], ['拟招生人数原文', r.quota_text], ['专项计划', r.special_plans.join(' / ')], ['采集时间', r.fetched_at]].map(([k, v]) =>
{k}{display(v)}
)}
{r.subjects.filter(Boolean).map((s, i) => {s})}
{r.remarks &&
{r.remarks}
}{r.reference_books?.length ?

参考书:{r.reference_books.join(';')}

: null}

历年分数线

{data.scores.map((s, i) => )}
年份口径总分政治外语业务一业务二备注
{s.year}{scopeName(s.scope)}{s.raw_code}{display(s.total)}{display(s.politics)}{display(s.english)}{display(s.special_one)}{display(s.special_two)}{s.note || '—'}

历年招生计划 / 目录快照

{data.plans.map((p, i) => )}
年份方向招生人数原文科目组合来源
{p.year}{display(p.direction)}{display(p.quota_text)}{p.subjects.filter(Boolean).join(' / ') || '未提供'}{p.source}
:
正在读取…
}
; +} + +function Trend({ scores }: { scores: History['scores'] }) { + const groups = scoreSeries(scores), points = groups.flatMap(g => g.points), colors = ['#14816f', '#4279ae', '#bb883c', '#965f93']; + if (!points.length) return
暂未获取该院系的历史分数线。采集历年考情后可查看趋势。
; + const years = [...new Set(points.map(p => p.year))].sort(), low = Math.floor((Math.min(...points.map(p => p.total)) - 10) / 10) * 10, high = Math.ceil((Math.max(...points.map(p => p.total)) + 10) / 10) * 10; + const x = (year: number) => years.length === 1 ? 380 : 55 + (year - years[0]) / (years[years.length - 1] - years[0]) * 650; + const y = (score: number) => 170 - (score - low) / (high - low) * 140; + return
{[low, (low + high) / 2, high].map(t => {t})}{years.map(t => {t})}{groups.map((g, i) => `${x(p.year)},${y(p.total)}`).join(' ')} />{g.points.map((p, j) => {p.total})})}
{groups.map((g, i) => {g.label})}
; +} + +function Tasks({ overview, majors, notify, refresh }: Shared) { + const [mode, setMode] = useState('combined'), [year, setYear] = useState(currentYear + 1), [major, setMajor] = useState(''), [province, setProvince] = useState(''), [school, setSchool] = useState(''), [years, setYears] = useState([currentYear - 2, currentYear - 1, currentYear]), [plans, setPlans] = useState(true), [jobs, setJobs] = useState([]), [logs, setLogs] = useState([]), [selected, setSelected] = useState(''), [busy, setBusy] = useState(false), [config, setConfig] = useState(null); + const load = useCallback(async () => { const data = await api('/jobs'); setJobs(data); setSelected(s => s || data[0]?.id || ''); }, []); + useEffect(() => { load().catch(e => notify(e.message, true)); api('/settings').then(setConfig).catch(e => notify(e.message, true)); }, [load, notify]); + const hasActive = jobs.some(j => activeStates.includes(j.status)); + useEffect(() => { if (!hasActive) return; const timer = setInterval(() => { load().catch(() => {}); refresh(); }, 2500); return () => clearInterval(timer); }, [hasActive, load, refresh]); + const job = jobs.find(j => j.id === selected); + const stamp = job?.updated_at; + useEffect(() => { if (!selected) return; const controller = new AbortController(); api(`/jobs/${selected}/logs`, { signal: controller.signal }).then(setLogs).catch(e => { if (e.name !== 'AbortError') notify(e.message, true); }); return () => controller.abort(); }, [selected, stamp, notify]); + async function start(e: React.FormEvent) { e.preventDefault(); setBusy(true); try { const result = await post('/jobs', { mode, year, major_code: major, school_keyword: school, provinces: province ? [province] : [], score_years: years, include_plans: plans }); setSelected(result.id); await load(); notify('采集任务已启动'); } catch (e) { notify((e as Error).message, true); } finally { setBusy(false); } } + async function action(action: string) { if (!job) return; setBusy(true); try { await post(`/jobs/${job.id}/${action}`); await load(); refresh(); } catch (e) { notify((e as Error).message, true); } finally { setBusy(false); } } + return <>{config ? modeName(config.network_mode) : '读取网络策略…'}

新建采集

{mode !== 'catalog' && <>
{Array.from({ length: 8 }, (_, i) => currentYear - i).map(y => )}
}
研招网仅开放当前目录。任务开始时核验年份;历史目录可从文件导入。仅采集历年考情时,使用已存目录或指定专业的院校索引。

首次使用,请先在「设置与登录」完成学信网登录。

任务记录

{jobs.length ? jobs.map(j => ) :
还没有任务。填写左侧条件开始采集。
}

运行日志

{job &&
{activeStates.includes(job.status) && }{resumeStates.includes(job.status) && }{!['completed', 'cancelled'].includes(job.status) && }
}
{job ? <>
状态 {statusName(job.status)}完成单元 {job.completed_units}保存记录 {job.records}失败单元 {job.errors}
{job.message}
{logs.map(l =>
{l.message}
)}{!logs.length && 暂无日志}
:
选择一个任务查看进度和日志。
}
; +} + +function ProxyPanel({ notify }: { notify: Notify }) { + const [data, setData] = useState(null), [busy, setBusy] = useState(false), [output, setOutput] = useState(''); + const load = useCallback(async () => setData(await api('/proxy?refresh=true')), []); + useEffect(() => { load().catch(e => notify(e.message, true)); }, [load, notify]); + async function operate(action: string) { setBusy(true); try { if (action === 'rotate') { setData(await post('/proxy/rotate')); notify('已切换当前路由'); } else if (action === 'refresh') await load(); else { const result = await post<{ output: string; ok: boolean }>('/proxy/manage', { action }); setOutput(result.output); notify(result.ok ? '代理池操作完成' : '代理池返回异常,请查看输出', !result.ok); await load(); } } catch (e) { notify((e as Error).message, true); } finally { setBusy(false); } } + return <>
} sub="以代理池 API 返回为准" />} sub="用于两个 HTTPS 数据源" />} sub="连接失败会切换其他候选" />} sub="保留池记录,修改配置可重置" />
当前网络路由 · {data ? modeName(data.mode) : '加载中'}

{data?.route === 'direct' ? '直接连接' : data?.route === 'pending' ? '等待可用代理' : data?.route || '正在读取代理池…'}

{data?.fallback ? '代理池暂无可用 IP,优先代理模式已回退直连。' : '登录、招生目录和历年考情共用该路由。'}

{data?.fallback ? '直连回退' : data?.active_proxy ? '代理已选择' : '当前策略'}
{data?.error &&
{data.error}
}

可用代理列表

{data?.proxies.length || 0} 个候选
{data?.proxies.length ? data.proxies.map(p =>
{p.url}{p.failed ? '已隔离' : p.active ? '当前 IP' : '池内可选'}
) :
没有 HTTPS 候选。可启动代理池抓取服务,或在设置中添加备用 IP。
}

本机代理池服务

通过现有项目的 manage.ps1 控制 Redis、API 与抓取调度器。

{busy &&

正在处理…

}{output &&
{output}
}
切换 IP 保留原代理;连接失败仅在本次工作台运行中隔离。代理池的持续可用性由其检测服务维护。
; +} + +function SettingsPanel({ notify }: { notify: Notify }) { + const [config, setConfig] = useState(null), [auth, setAuth] = useState(null), [backup, setBackup] = useState(''), [cookie, setCookie] = useState(''), [username, setUsername] = useState(''), [password, setPassword] = useState(''), [busy, setBusy] = useState(''); + useEffect(() => { Promise.all([api('/settings'), api('/auth')]).then(([c, a]) => { setConfig(c); setBackup(c.backup_proxies.join('\n')); setAuth(a); }).catch(e => notify(e.message, true)); }, [notify]); + const browserWaiting = auth && ['opening', 'waiting'].includes(auth.browser_state); + useEffect(() => { if (!browserWaiting) return; const timer = setInterval(() => api('/auth').then(setAuth).catch(() => {}), 2000); return () => clearInterval(timer); }, [browserWaiting]); + async function save(e: React.FormEvent) { e.preventDefault(); setBusy('save'); try { setConfig(await api('/settings', { method: 'PUT', body: JSON.stringify({ ...config, backup_proxies: backup.split('\n').filter(Boolean) }) })); notify('网络设置已保存'); } catch (e) { notify((e as Error).message, true); } finally { setBusy(''); } } + async function authenticate(action: string) { setBusy(action); try { const result = await post('/auth/' + action, action === 'cookie' ? { cookie } : action === 'login' ? { username, password } : {}); setAuth(result); notify(action === 'browser' ? result.browser_message : result.message, action === 'check' && !result.valid); } catch (e) { notify((e as Error).message, true); } finally { setBusy(''); if (action === 'cookie') setCookie(''); if (action === 'login') setPassword(''); } } + function update(key: keyof Config, value: string | number) { setConfig(c => c ? { ...c, [key]: value } : c); } + return <>

网络与代理设置

{config ?