refactor: refactor sur server et les pages qui commencer à faire beaucoup trop de ligne
This commit is contained in:
1 parent
609ddc777f
commit
bf2274b761
25 files changed
+2769
-2725
No files matched your search
@@ -0,0 +1,65 @@
|
||||
import { DATA_DIR } from '../config.js'
|
||||
import path from 'path'
|
||||
import fs from 'fs'
|
||||
|
||||
const GITHUB_BASE = process.env.GITHUB_BASE
|
||||
?? 'https://raw.githubusercontent.com/masutayunikon/fankarr-scraper/main'
|
||||
|
||||
const CACHE_TTL_MS = 6 * 60 * 60 * 1000
|
||||
|
||||
interface CacheEntry<T> { data: T; expiresAt: number }
|
||||
const _cache = new Map<string, CacheEntry<any>>()
|
||||
|
||||
export function cacheGet<T>(key: string): T | null {
|
||||
const entry = _cache.get(key)
|
||||
if (!entry) return null
|
||||
if (Date.now() > entry.expiresAt) { _cache.delete(key); return null }
|
||||
return entry.data as T
|
||||
}
|
||||
|
||||
export function cacheSet<T>(key: string, data: T): void {
|
||||
_cache.set(key, { data, expiresAt: Date.now() + CACHE_TTL_MS })
|
||||
}
|
||||
|
||||
export function cacheClear(): void { _cache.clear() }
|
||||
|
||||
export function cacheSize(): number { return _cache.size }
|
||||
|
||||
export async function githubGet(urlPath: string): Promise<any> {
|
||||
const cached = cacheGet(urlPath)
|
||||
if (cached) return cached
|
||||
const res = await fetch(`${GITHUB_BASE}/${urlPath}`)
|
||||
if (!res.ok) throw new Error(`GitHub ${res.status}: ${urlPath}`)
|
||||
const data = await res.json()
|
||||
cacheSet(urlPath, data)
|
||||
return data
|
||||
}
|
||||
|
||||
export async function readAvailable(): Promise<number[]> {
|
||||
try { return await githubGet('available.json') as number[] }
|
||||
catch { return [] }
|
||||
}
|
||||
|
||||
export async function readInfohashMap(): Promise<Record<string, string>> {
|
||||
try { return await githubGet('infohash_map.json') as Record<string, string> }
|
||||
catch { return {} }
|
||||
}
|
||||
|
||||
export async function readSerieData(serieId: number): Promise<any | null> {
|
||||
try { return await githubGet(`series/${serieId}.json`) }
|
||||
catch { return null }
|
||||
}
|
||||
|
||||
export async function loadEnrichedSeriesData(): Promise<any[]> {
|
||||
const ids = await readAvailable()
|
||||
const allSd = await Promise.all(ids.map(id => readSerieData(id)))
|
||||
return allSd.filter(Boolean)
|
||||
}
|
||||
|
||||
export function loadOrganized(): Record<string, Record<string, any>> {
|
||||
try {
|
||||
const p = path.join(DATA_DIR, 'organized.json')
|
||||
if (fs.existsSync(p)) return JSON.parse(fs.readFileSync(p, 'utf-8'))
|
||||
} catch {}
|
||||
return {}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { DATA_DIR } from '../config.js'
|
||||
import { readSettings } from '../settings.js'
|
||||
import { logger } from '../logger.js'
|
||||
import { getGitlabTitle } from '../gitlab-map.js'
|
||||
import { loadEnrichedSeriesData, loadOrganized } from './github-cache.js'
|
||||
|
||||
export const GITLAB_API_NFO = 'https://gitlab.com/api/v4/projects/ElPouki%2Ffankai_pack/repository'
|
||||
export const GITLAB_RAW_NFO = 'https://gitlab.com/ElPouki/fankai_pack/-/raw/main/pack'
|
||||
|
||||
export interface NfoUpdateNotif {
|
||||
serieTitle: string; commitSha: string; commitMsg: string
|
||||
updatedAt: string; status: 'updated' | 'error'; error?: string
|
||||
}
|
||||
|
||||
export const recentNfoUpdates: NfoUpdateNotif[] = []
|
||||
const MAX_NFO_NOTIFS = 30
|
||||
const NFO_COMMITS_PATH = path.join(DATA_DIR, 'nfo_commits.json')
|
||||
|
||||
export function loadNfoCommits(): Record<string, string> {
|
||||
try { if (!fs.existsSync(NFO_COMMITS_PATH)) return {}; return JSON.parse(fs.readFileSync(NFO_COMMITS_PATH, 'utf-8')) }
|
||||
catch { return {} }
|
||||
}
|
||||
|
||||
export function saveNfoCommits(commits: Record<string, string>) {
|
||||
fs.mkdirSync(path.dirname(NFO_COMMITS_PATH), { recursive: true })
|
||||
fs.writeFileSync(NFO_COMMITS_PATH, JSON.stringify(commits, null, 2), 'utf-8')
|
||||
}
|
||||
|
||||
export async function getLatestGitlabCommit(gitlabTitle: string): Promise<{ sha: string; message: string } | null> {
|
||||
try {
|
||||
const res = await fetch(`${GITLAB_API_NFO}/commits?path=${encodeURIComponent('pack/' + gitlabTitle)}&per_page=1&ref_name=main`, { headers: { 'User-Agent': 'fankarr' } })
|
||||
if (!res.ok) return null
|
||||
const data: any[] = await res.json()
|
||||
if (!data.length) return null
|
||||
return { sha: data[0].id, message: data[0].title ?? data[0].message ?? '' }
|
||||
} catch { return null }
|
||||
}
|
||||
|
||||
export async function downloadNfoFolder(gitlabTitle: string, destRoot: string): Promise<void> {
|
||||
let files: any[] = [], page = 1
|
||||
while (true) {
|
||||
const batch = await fetch(`${GITLAB_API_NFO}/tree?path=${encodeURIComponent('pack/' + gitlabTitle)}&recursive=true&per_page=100&page=${page}&ref=main`, { headers: { 'User-Agent': 'fankarr' } }).then(r => r.ok ? r.json() : [])
|
||||
if (!Array.isArray(batch) || batch.length === 0) break
|
||||
files.push(...batch); if (batch.length < 100) break; page++
|
||||
}
|
||||
let downloaded = 0, skipped = 0
|
||||
for (const entry of files.filter((f: any) => f.type === 'blob')) {
|
||||
const relativePath = entry.path.replace(`pack/${gitlabTitle}/`, '')
|
||||
const destPath = path.join(destRoot, relativePath)
|
||||
try {
|
||||
const res = await fetch(`${GITLAB_RAW_NFO}/${encodeURIComponent(gitlabTitle)}/${relativePath.split('/').map(encodeURIComponent).join('/')}`, { headers: { 'User-Agent': 'fankarr' } })
|
||||
if (!res.ok) { skipped++; continue }
|
||||
fs.mkdirSync(path.dirname(destPath), { recursive: true })
|
||||
fs.writeFileSync(destPath, Buffer.from(await res.arrayBuffer()))
|
||||
downloaded++
|
||||
} catch { skipped++ }
|
||||
}
|
||||
logger.info('nfo-update', `"${gitlabTitle}" — ${downloaded} fichiers mis à jour, ${skipped} ignorés`)
|
||||
}
|
||||
|
||||
export async function checkNfoUpdates() {
|
||||
const { mediaPath, nfoSupport } = readSettings()
|
||||
if (!nfoSupport) return
|
||||
const organized = loadOrganized()
|
||||
if (Object.keys(organized).length === 0) return
|
||||
const seriesData = await loadEnrichedSeriesData()
|
||||
const knownHashes = new Set(Object.keys(organized))
|
||||
const serieTitles = new Set<string>()
|
||||
for (const sd of seriesData) {
|
||||
const allTorrents = [...(sd.torrents ?? []), ...(sd.seasons ?? []).flatMap((s: any) => s.torrents ?? []), ...(sd.seasons ?? []).flatMap((s: any) => (s.episodes ?? []).flatMap((e: any) => e.torrents ?? []))]
|
||||
if (allTorrents.some((t: any) => knownHashes.has(t.infohash?.toLowerCase()))) {
|
||||
const rawTitle = sd.title ?? sd.show_title ?? ''
|
||||
if (rawTitle) serieTitles.add(rawTitle.replace(/:/g, ' -').replace(/[<>"/\\|?*]/g, '').replace(/\s+/g, ' ').trim())
|
||||
}
|
||||
}
|
||||
if (serieTitles.size === 0) return
|
||||
const commits = loadNfoCommits()
|
||||
let hasChanges = false
|
||||
for (const serieTitle of serieTitles) {
|
||||
const gitlabTitle = getGitlabTitle(serieTitle)
|
||||
const latest = await getLatestGitlabCommit(gitlabTitle)
|
||||
if (!latest || commits[gitlabTitle] === latest.sha) continue
|
||||
logger.info('nfo-update', `MAJ NFO détectée pour "${gitlabTitle}" (${latest.sha.slice(0, 8)})`)
|
||||
try {
|
||||
await downloadNfoFolder(gitlabTitle, path.join(mediaPath, serieTitle))
|
||||
commits[gitlabTitle] = latest.sha
|
||||
hasChanges = true
|
||||
recentNfoUpdates.unshift({ serieTitle, commitSha: latest.sha.slice(0, 8), commitMsg: latest.message, updatedAt: new Date().toISOString(), status: 'updated' })
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Erreur inconnue'
|
||||
logger.error('nfo-update', `Échec MAJ NFO "${gitlabTitle}" : ${msg}`)
|
||||
recentNfoUpdates.unshift({ serieTitle, commitSha: latest.sha.slice(0, 8), commitMsg: latest.message, updatedAt: new Date().toISOString(), status: 'error', error: msg })
|
||||
}
|
||||
if (recentNfoUpdates.length > MAX_NFO_NOTIFS) recentNfoUpdates.pop()
|
||||
}
|
||||
if (hasChanges) saveNfoCommits(commits)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
export interface OrganizeNotif {
|
||||
hash: string; name: string; done: number; skipped: number; errors: number
|
||||
errorFiles: { file: string; error: string }[]; at: string
|
||||
}
|
||||
|
||||
export const recentOrganized: OrganizeNotif[] = []
|
||||
const MAX_NOTIFS = 20
|
||||
|
||||
export function pushNotif(n: OrganizeNotif) {
|
||||
recentOrganized.unshift(n)
|
||||
if (recentOrganized.length > MAX_NOTIFS) recentOrganized.pop()
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
const FANKAI_API = 'https://metadata.fankai.fr'
|
||||
|
||||
export async function fankaiGet(endpoint: string): Promise<any> {
|
||||
const res = await fetch(`${FANKAI_API}${endpoint}`)
|
||||
if (!res.ok) throw new Error(`Fankai API ${res.status}: ${endpoint}`)
|
||||
return res.json()
|
||||
}
|
||||
|
||||
export function imgProxy(url: string | null | undefined, w: number, q = 70): string | null {
|
||||
if (!url) return null
|
||||
const stripped = url.replace(/^https?:\/\//, '')
|
||||
return `https://wsrv.nl/?url=${stripped}&w=${w}&q=${q}`
|
||||
}
|
||||
|
||||
export function normalizeSerie(serie: any): any {
|
||||
return { ...serie, poster_image: imgProxy(serie.poster_image ?? serie.images?.poster ?? null, 300), fanart_image: imgProxy(serie.fanart_image ?? serie.images?.fanart ?? null, 1280, 60) }
|
||||
}
|
||||
|
||||
export function normalizeSeason(season: any): any {
|
||||
return { ...season, poster_image: imgProxy(season.poster_image ?? season.images?.poster ?? null, 300) }
|
||||
}
|
||||
|
||||
export function normalizeEpisode(ep: any): any {
|
||||
return { ...ep, thumb_image: imgProxy(ep.thumb_image ?? ep.thumbnail ?? null, 400) }
|
||||
}
|
||||
|
||||
export function extractTorrentsFromSerieData(sd: any): any[] {
|
||||
const result: any[] = []
|
||||
if (!sd) return result
|
||||
for (const t of sd.torrents ?? []) {
|
||||
result.push({ ...t, raw: t.title, type: 'pack_integrale', serie_id: sd.id, serie_title: sd.title, _serieLevel: true, _serieData: sd, _torrentIdx: (sd.torrents ?? []).indexOf(t) })
|
||||
}
|
||||
for (const season of sd.seasons ?? []) {
|
||||
for (const t of season.torrents ?? []) {
|
||||
result.push({ ...t, raw: t.title, type: 'pack_saison', serie_id: sd.id, serie_title: sd.title, season_id: season.id, season_number: season.season_number, _seasonLevel: true, _season: season, _torrentIdx: (season.torrents ?? []).indexOf(t) })
|
||||
}
|
||||
for (const ep of season.episodes ?? []) {
|
||||
for (const t of ep.torrents ?? []) {
|
||||
result.push({ ...t, raw: t.title, type: 'episode', serie_id: sd.id, serie_title: sd.title, season_id: season.id, season_number: season.season_number, episode_id: ep.id, episode_number: ep.episode_number, _episodeLevel: true, _episode: ep, _torrentIdx: (ep.torrents ?? []).indexOf(t) })
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export function buildResolvedEpisodes(sd: any, hash: string, seasonFilter?: number): any[] {
|
||||
const resolved: any[] = []
|
||||
const h = hash.toLowerCase()
|
||||
for (const season of sd.seasons ?? []) {
|
||||
if (seasonFilter !== undefined && season.season_number !== seasonFilter) continue
|
||||
for (const ep of season.episodes ?? []) {
|
||||
const paths: any[] = ep.paths ?? []
|
||||
let filePath: string | null = null
|
||||
for (const p of paths) {
|
||||
if (typeof p === 'string') { if (!filePath) filePath = p.replace(/\\/g, '/') }
|
||||
else if (p?.infohash?.toLowerCase() === h) { filePath = p.path.replace(/\\/g, '/'); break }
|
||||
}
|
||||
if (!filePath) continue
|
||||
const filename = filePath.split('/').pop() ?? filePath
|
||||
resolved.push({ episode_id: ep.id, episode_number: ep.episode_number, season_number: season.season_number, season_id: season.id, filename, original_filename: filename })
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
export function computeSerieDownloadState(serieData: any | null, organized: Record<string, Record<string, any>>, activeTorrents: Set<string>): 'none' | 'downloading' | 'partial' | 'complete' {
|
||||
if (!serieData) return 'none'
|
||||
const allTorrents = extractTorrentsFromSerieData(serieData)
|
||||
|
||||
if (allTorrents.some(t => t.infohash && activeTorrents.has(t.infohash.toLowerCase()))) return 'downloading'
|
||||
|
||||
const allEpisodeIds = new Set<number>()
|
||||
for (const season of serieData.seasons ?? []) {
|
||||
for (const ep of season.episodes ?? []) {
|
||||
allEpisodeIds.add(ep.id)
|
||||
}
|
||||
}
|
||||
|
||||
const organizedEpisodeIds = new Set<number>()
|
||||
|
||||
for (const t of allTorrents) {
|
||||
const hash = t.infohash?.toLowerCase()
|
||||
if (!hash) continue
|
||||
const orgFiles = organized[hash] ?? {}
|
||||
if (Object.keys(orgFiles).length === 0) continue
|
||||
for (const ep of buildResolvedEpisodes(serieData, hash, t.season_number)) {
|
||||
const isOrganized = orgFiles[String(ep.episode_id)] !== undefined || orgFiles[ep.filename] !== undefined
|
||||
if (isOrganized) organizedEpisodeIds.add(ep.episode_id)
|
||||
}
|
||||
}
|
||||
|
||||
const manualOrg = organized['manual'] ?? {}
|
||||
for (const season of serieData.seasons ?? []) {
|
||||
for (const ep of season.episodes ?? []) {
|
||||
if (manualOrg[String(ep.id)]) organizedEpisodeIds.add(ep.id)
|
||||
}
|
||||
}
|
||||
|
||||
if (allEpisodeIds.size === 0 || organizedEpisodeIds.size === 0) return 'none'
|
||||
if (organizedEpisodeIds.size >= allEpisodeIds.size) return 'complete'
|
||||
return 'partial'
|
||||
}
|
||||
Reference in new issue
Block a user