chore: use fuman utils and fetch

This commit is contained in:
alina 🌸 2024-11-28 22:56:55 +03:00
parent dc40678f96
commit aaa0e20f3d
Signed by: teidesu
SSH key fingerprint: SHA256:uNeCpw6aTSU4aIObXLvHfLkDa82HWH9EiOj9AXOIRpI
15 changed files with 201 additions and 272 deletions

View file

@ -15,6 +15,8 @@
"@astrojs/check": "^0.9.1",
"@astrojs/node": "^8.3.2",
"@astrojs/solid-js": "^4.4.0",
"@fuman/fetch": "https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb",
"@fuman/utils": "https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb",
"@mtcute/dispatcher": "^0.16.0",
"@mtcute/node": "^0.16.3",
"@tanstack/solid-query": "^5.51.21",

View file

@ -17,6 +17,12 @@ importers:
'@astrojs/solid-js':
specifier: ^4.4.0
version: 4.4.0(solid-js@1.8.19)(vite@5.3.5(@types/node@22.0.2)(sugarss@4.0.1(postcss@8.4.40)))
'@fuman/fetch':
specifier: https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb
version: https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb(zod@3.23.8)
'@fuman/utils':
specifier: https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb
version: https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb
'@mtcute/dispatcher':
specifier: ^0.16.0
version: 0.16.0
@ -819,6 +825,28 @@ packages:
resolution: {integrity: sha512-BsWiH1yFGjXXS2yvrf5LyuoSIIbPrGUWob917o+BTKuZ7qJdxX8aJLRxs1fS9n6r7vESrq1OUqb68dANcFXuQQ==}
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
'@fuman/fetch@https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb':
resolution: {tarball: https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb}
version: 0.0.1
peerDependencies:
tough-cookie: ^5.0.0 || ^4.0.0
valibot: ^0.42.0
yup: ^1.0.0
zod: ^3.0.0
peerDependenciesMeta:
tough-cookie:
optional: true
valibot:
optional: true
yup:
optional: true
zod:
optional: true
'@fuman/utils@https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb':
resolution: {tarball: https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb}
version: 0.0.1
'@humanwhocodes/module-importer@1.0.1':
resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==}
engines: {node: '>=12.22'}
@ -4438,6 +4466,14 @@ snapshots:
'@eslint/object-schema@2.1.4': {}
'@fuman/fetch@https://pkg.pr.new/teidesu/fuman/@fuman/fetch@b0c74cb(zod@3.23.8)':
dependencies:
'@fuman/utils': https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb
optionalDependencies:
zod: 3.23.8
'@fuman/utils@https://pkg.pr.new/teidesu/fuman/@fuman/utils@b0c74cb': {}
'@humanwhocodes/module-importer@1.0.1': {}
'@humanwhocodes/retry@0.3.0': {}

View file

@ -3,7 +3,14 @@ import { html } from '@mtcute/node'
import parseDuration from 'parse-duration'
import { env } from '../env'
import { answerBySerial, approveShout, banShouts, declineShout, deleteBySerial, unbanShouts } from '../service/shoutbox'
import {
answerBySerial,
approveShout,
banShouts,
declineShout,
deleteBySerial,
unbanShouts,
} from '../service/shoutbox'
export const ShoutboxAction = new CallbackDataBuilder('shoutbox', 'id', 'action')

View file

@ -1,8 +1,8 @@
import { z } from 'zod'
import { AsyncResource } from '@fuman/utils'
import { Reloadable } from '../utils/reloadable'
import { env } from '../env'
import { zodValidate } from '../../utils/zod'
import { ffetch } from '../utils/fetch.ts'
export const AVAILABLE_CURRENCIES = ['RUB', 'USD', 'EUR']
const TTL = 60 * 60 * 1000 // 1 hour
@ -17,25 +17,24 @@ const schema = z.object({
})),
})
const reloadable = new Reloadable({
name: 'currencies',
expiresIn: () => TTL,
async fetch() {
const reloadable = new AsyncResource<z.infer<typeof schema>>({
// expiresIn: () => TTL,
async fetcher() {
// https://api.currencyapi.com/v3/latest?apikey=cur_live_ZGgJCl3CfMM7TqXSdlUTiKlO2e81lLcOVX5mCXb6&currencies=USD%2CEUR
// apikey=cur_live_ZGgJCl3CfMM7TqXSdlUTiKlO2e81lLcOVX5mCXb6&currencies=USD%2CEUR
const res = await fetch(`https://api.currencyapi.com/v3/latest?${new URLSearchParams({
apikey: env.CURRENCY_API_TOKEN,
currencies: AVAILABLE_CURRENCIES.slice(1).join(','),
base_currency: AVAILABLE_CURRENCIES[0],
})}`)
const res = await ffetch('https://api.currencyapi.com/v3/latest', {
query: {
apikey: env.CURRENCY_API_TOKEN,
currencies: AVAILABLE_CURRENCIES.slice(1).join(','),
base_currency: AVAILABLE_CURRENCIES[0],
},
}).parsedJson(schema)
if (!res.ok) {
throw new Error(`Failed to fetch currencies: ${res.status} ${await res.text()}`)
return {
data: res,
expiresIn: TTL,
}
return zodValidate(schema, await res.json())
},
lazy: true,
swr: true,
})

View file

@ -1,49 +1,49 @@
import { z } from 'zod'
// import { z } from 'zod'
import { Reloadable } from '~/backend/utils/reloadable'
import { zodValidate } from '~/utils/zod'
// import { Reloadable } from '~/backend/utils/reloadable'
// import { zodValidate } from '~/utils/zod'
const ENDPOINT = 'https://very.stupid.fish/api/users/notes'
const TTL = 3 * 60 * 60 * 1000 // 3 hours
const STALE_TTL = 8 * 60 * 60 * 1000 // 8 hours
const BODY = {
userId: '9o5tqc3ok6pf5hjx',
withRenotes: false,
withReplies: false,
withChannelNotes: false,
withFiles: false,
limit: 1,
allowPartial: true,
}
// const ENDPOINT = 'https://very.stupid.fish/api/users/notes'
// const TTL = 3 * 60 * 60 * 1000 // 3 hours
// const STALE_TTL = 8 * 60 * 60 * 1000 // 8 hours
// const BODY = {
// userId: '9o5tqc3ok6pf5hjx',
// withRenotes: false,
// withReplies: false,
// withChannelNotes: false,
// withFiles: false,
// limit: 1,
// allowPartial: true,
// }
const schema = z.object({
id: z.string(),
createdAt: z.string(),
updatedAt: z.string().nullable().optional(),
text: z.nullable(z.string()),
})
// const schema = z.object({
// id: z.string(),
// createdAt: z.string(),
// updatedAt: z.string().nullable().optional(),
// text: z.nullable(z.string()),
// })
export const fediLastSeen = new Reloadable<z.infer<typeof schema>>({
name: 'fedi-last-seen',
async fetch() {
const res = await fetch(ENDPOINT, {
method: 'POST',
body: JSON.stringify(BODY),
headers: {
'Content-Type': 'application/json',
},
})
// export const fediLastSeen = new Reloadable<z.infer<typeof schema>>({
// name: 'fedi-last-seen',
// async fetch() {
// const res = await fetch(ENDPOINT, {
// method: 'POST',
// body: JSON.stringify(BODY),
// headers: {
// 'Content-Type': 'application/json',
// },
// })
if (!res.ok) {
throw new Error(`Failed to fetch fedi last seen: ${res.status} ${await res.text()}`)
}
// if (!res.ok) {
// throw new Error(`Failed to fetch fedi last seen: ${res.status} ${await res.text()}`)
// }
const data = await zodValidate(z.array(schema), await res.json())
// const data = await zodValidate(z.array(schema), await res.json())
return data[0]
},
expiresIn: () => TTL,
lazy: true,
swr: true,
swrValidator: (_data, time) => Date.now() - time < STALE_TTL,
})
// return data[0]
// },
// expiresIn: () => TTL,
// lazy: true,
// swr: true,
// swrValidator: (_data, time) => Date.now() - time < STALE_TTL,
// })

View file

@ -1,7 +1,7 @@
import { AsyncResource } from '@fuman/utils'
import { z } from 'zod'
import { Reloadable } from '~/backend/utils/reloadable'
import { zodValidate } from '~/utils/zod'
import { ffetch } from '../../utils/fetch.ts'
const ENDPOINT = 'https://api.github.com/users/teidesu/events/public?per_page=1'
const TTL = 1 * 60 * 60 * 1000 // 1 hour
@ -16,26 +16,20 @@ const schema = z.object({
created_at: z.string(),
})
export const githubLastSeen = new Reloadable<z.infer<typeof schema>>({
name: 'github-last-seen',
async fetch() {
const res = await fetch(ENDPOINT, {
export const githubLastSeen = new AsyncResource<z.infer<typeof schema>>({
async fetcher() {
const res = await ffetch(ENDPOINT, {
headers: {
'User-Agent': 'tei.su/1.0',
'X-GitHub-Api-Version': '2022-11-28',
},
})
}).parsedJson(z.array(schema))
if (!res.ok) {
throw new Error(`Failed to fetch github last seen: ${res.status} ${await res.text()}`)
return {
data: res[0],
expiresIn: TTL,
}
const data = await zodValidate(z.array(schema), await res.json())
return data[0]
},
expiresIn: () => TTL,
lazy: true,
swr: true,
swrValidator: (_data, time) => Date.now() - time < STALE_TTL,
swrValidator: ({ currentFetchedAt }) => Date.now() - currentFetchedAt < STALE_TTL,
})

View file

@ -1,4 +1,3 @@
import { fediLastSeen } from './fedi'
import { githubLastSeen } from './github'
import { lastfm } from './lastfm'
import { shikimoriLastSeen } from './shikimori'
@ -15,12 +14,12 @@ export interface LastSeenItem {
export async function fetchLastSeen() {
const [
lastfmData,
fediData,
// fediData,
shikimoriData,
githubData,
] = await Promise.all([
lastfm.get(),
fediLastSeen.get(),
// fediLastSeen.get(),
shikimoriLastSeen.get(),
githubLastSeen.get(),
])
@ -37,15 +36,15 @@ export async function fetchLastSeen() {
})
}
if (fediData) {
res.push({
source: 'fedi',
sourceLink: 'https://very.stupid.fish/@teidesu',
time: new Date(fediData.updatedAt ?? fediData.createdAt).getTime(),
text: fediData.text?.slice(0, 40) || '[no text]',
link: `https://very.stupid.fish/notes/${fediData.id}`,
})
}
// if (fediData) {
// res.push({
// source: 'fedi',
// sourceLink: 'https://very.stupid.fish/@teidesu',
// time: new Date(fediData.updatedAt ?? fediData.createdAt).getTime(),
// text: fediData.text?.slice(0, 40) || '[no text]',
// link: `https://very.stupid.fish/notes/${fediData.id}`,
// })
// }
if (shikimoriData) {
// thx morr for this fucking awesome api

View file

@ -1,8 +1,9 @@
import { z } from 'zod'
import { AsyncResource } from '@fuman/utils'
import { Reloadable } from '~/backend/utils/reloadable'
import { zodValidate } from '~/utils/zod'
import { env } from '~/backend/env'
import { ffetch } from '../../utils/fetch.ts'
const LASTFM_TTL = 1000 * 60 * 5 // 5 minutes
const LASTFM_STALE_TTL = 1000 * 60 * 60 // 1 hour
@ -33,39 +34,31 @@ const ResponseSchema = z.object({
}),
})
export const lastfm = new Reloadable<LastfmTrack>({
name: 'last-track',
async fetch(prev) {
const params = new URLSearchParams({
method: 'user.getrecenttracks',
user: LASTFM_USERNAME,
api_key: LASTFM_TOKEN,
format: 'json',
limit: '1',
})
if (prev?.date) {
params.set('from', prev.date!.uts)
}
const res = await fetch(`https://ws.audioscrobbler.com/2.0/?${params}`)
export const lastfm = new AsyncResource<LastfmTrack>({
async fetcher({ current }) {
const res = await ffetch('https://ws.audioscrobbler.com/2.0/', {
query: {
method: 'user.getrecenttracks',
user: LASTFM_USERNAME,
api_key: LASTFM_TOKEN,
format: 'json',
limit: '1',
from: current?.date?.uts,
},
}).parsedJson(ResponseSchema)
if (!res.ok) {
throw new Error(`Failed to fetch last.fm data: ${res.status} ${await res.text()}`)
}
const data = await res.json()
const parsed = await zodValidate(ResponseSchema, data)
const track = parsed.recenttracks.track[0]
const track = res.recenttracks.track[0]
if (!track.date && track['@attr']?.nowplaying) {
track.date = { uts: Math.floor(Date.now() / 1000).toString() }
} else if (!track.date) {
throw new Error('no track found')
}
return track
return {
data: track,
expiresIn: LASTFM_TTL,
}
},
expiresIn: () => LASTFM_TTL,
lazy: true,
swr: true,
swrValidator: (_data, time) => Date.now() - time < LASTFM_STALE_TTL,
swrValidator: ({ currentExpiresAt }) => Date.now() - currentExpiresAt < LASTFM_STALE_TTL,
})

View file

@ -1,7 +1,7 @@
import { AsyncResource } from '@fuman/utils'
import { z } from 'zod'
import { Reloadable } from '~/backend/utils/reloadable'
import { zodValidate } from '~/utils/zod'
import { ffetch } from '../../utils/fetch.ts'
const ENDPOINT = 'https://shikimori.one/api/users/698215/history?limit=1'
const TTL = 3 * 60 * 60 * 1000 // 3 hours
@ -16,25 +16,15 @@ const schema = z.object({
}),
})
export const shikimoriLastSeen = new Reloadable<z.infer<typeof schema>>({
name: 'shikimori-last-seen',
async fetch() {
const res = await fetch(ENDPOINT, {
headers: {
'User-Agent': 'tei.su/1.0',
},
})
export const shikimoriLastSeen = new AsyncResource<z.infer<typeof schema>>({
async fetcher() {
const res = await ffetch(ENDPOINT).parsedJson(z.array(schema))
if (!res.ok) {
throw new Error(`Failed to fetch shikimori last seen: ${res.status} ${await res.text()}`)
return {
data: res[0],
expiresIn: TTL,
}
const data = await zodValidate(z.array(schema), await res.json())
return data[0]
},
expiresIn: () => TTL,
lazy: true,
swr: true,
swrValidator: (_data, time) => Date.now() - time < STALE_TTL,
swrValidator: ({ currentFetchedAt }) => Date.now() - currentFetchedAt < STALE_TTL,
})

View file

@ -1,22 +1,38 @@
import { ffetchAddons, ffetchBase } from '@fuman/fetch'
import { ffetchZodAdapter } from '@fuman/fetch/zod'
import { z } from 'zod'
import { isBotUserAgent } from '../utils/bot'
import { env } from '~/backend/env'
const ffetch = ffetchBase.extend({
addons: [
ffetchAddons.parser(ffetchZodAdapter()),
ffetchAddons.timeout(),
],
baseUrl: env.UMAMI_HOST,
timeout: 1000,
})
export async function umamiFetchStats(page: string, startAt: number) {
if (import.meta.env.DEV) {
return Promise.resolve({ visitors: { value: 1337 } })
}
const res = await fetch(`${env.UMAMI_HOST}/api/websites/${env.UMAMI_SITE_ID}/stats?${new URLSearchParams({
endAt: Math.floor(Date.now()).toString(),
startAt: startAt.toString(),
url: page,
})}`, {
return await ffetch(`/api/websites/${env.UMAMI_SITE_ID}/stats`, {
query: {
endAt: Math.floor(Date.now()).toString(),
startAt: startAt.toString(),
url: page,
},
headers: {
Authorization: `Bearer ${env.UMAMI_TOKEN}`,
},
})
return await res.json()
}).parsedJson(z.object({
visitors: z.object({
value: z.number(),
}),
}))
}
export function umamiLogThisVisit(request: Request, path?: string, website = env.UMAMI_SITE_ID): void {
@ -24,8 +40,8 @@ export function umamiLogThisVisit(request: Request, path?: string, website = env
if (isBotUserAgent(request.headers.get('user-agent') || '')) return
const language = request.headers.get('accept-language')?.split(';')[0].split(',')[0] || ''
fetch(`${env.UMAMI_HOST}/api/send`, {
body: JSON.stringify({
ffetch.post('/api/send', {
json: {
payload: {
hostname: request.headers.get('host') || '',
language,
@ -36,13 +52,11 @@ export function umamiLogThisVisit(request: Request, path?: string, website = env
website,
},
type: 'event',
}),
},
headers: {
'Content-Type': 'application/json',
'User-Agent': request.headers.get('user-agent') || '',
'X-Forwarded-For': request.headers.get('x-forwarded-for')?.[0] || '',
},
method: 'POST',
}).then(async (r) => {
if (!r.ok) throw new Error(`failed to log visit: ${r.status} ${await r.text()}`)
}).catch((err) => {

View file

@ -1,7 +1,7 @@
import { AsyncResource } from '@fuman/utils'
import { z } from 'zod'
import { Reloadable } from '~/backend/utils/reloadable'
import { zodValidate } from '~/utils/zod'
import { ffetch } from '../utils/fetch.ts'
const WEBRING_URL = 'https://otomir23.me/webring/5/data'
const WEBRING_TTL = 1000 * 60 * 60 * 24 // 24 hours
@ -19,21 +19,14 @@ const WebringData = z.object({
})
export type WebringData = z.infer<typeof WebringData>
export const webring = new Reloadable({
name: 'webring',
fetch: async () => {
const response = await fetch(WEBRING_URL)
if (!response.ok) {
const text = await response.text()
throw new Error(`Failed to fetch webring data: ${response.status} ${text}`)
export const webring = new AsyncResource<WebringData>({
fetcher: async () => {
const res = await ffetch(WEBRING_URL).parsedJson(WebringData)
return {
data: res,
expiresIn: WEBRING_TTL,
}
const data = await response.json()
const parsed = await zodValidate(WebringData, data)
return parsed
},
expiresIn: () => WEBRING_TTL,
lazy: true,
swr: true,
})

View file

@ -0,0 +1,12 @@
import { ffetchAddons, ffetchBase } from '@fuman/fetch'
import { ffetchZodAdapter } from '@fuman/fetch/zod'
export const ffetch = ffetchBase.extend({
addons: [
ffetchAddons.parser(ffetchZodAdapter()),
ffetchAddons.retry(),
],
headers: {
'User-Agent': 'tei.su/1.0',
},
})

View file

@ -1,25 +0,0 @@
/**
* A promise that can be resolved or rejected from outside.
*/
export type ControllablePromise<T = unknown> = Promise<T> & {
resolve: (val: T) => void
reject: (err?: unknown) => void
}
/**
* Creates a promise that can be resolved or rejected from outside.
*/
export function createControllablePromise<T = unknown>(): ControllablePromise<T> {
let _resolve: ControllablePromise<T>['resolve']
let _reject: ControllablePromise<T>['reject']
const promise = new Promise<T>((resolve, reject) => {
_resolve = resolve
_reject = reject
})
// ts doesn't like this, but it's fine
;(promise as ControllablePromise<T>).resolve = _resolve!
;(promise as ControllablePromise<T>).reject = _reject!
return promise as ControllablePromise<T>
}

View file

@ -1,83 +0,0 @@
import { type ControllablePromise, createControllablePromise } from './promise'
export interface ReloadableParams<T> {
name: string
// whether to avoid automatically reloading
lazy?: boolean
// whether to return old value while a new one is fetching
swr?: boolean
// if `swr` is enabled, whether the stale data can still be used
swrValidator?: (prev: T, prevTime: number) => boolean
fetch: (prev: T | null, prevTime: number) => Promise<T>
expiresIn: (data: T) => number
}
export class Reloadable<T> {
constructor(readonly params: ReloadableParams<T>) {}
private data: T | null = null
private lastFetchTime = 0
private expiresAt = 0
private updating?: ControllablePromise<void>
private timeout?: NodeJS.Timeout
async update(force = false): Promise<void> {
if (this.updating) {
await this.updating
return
}
if (!force && this.data && Date.now() < this.expiresAt) {
return
}
this.updating = createControllablePromise()
let result
try {
result = await this.params.fetch(this.data, this.lastFetchTime)
} catch (e) {
console.error(`Failed to fetch ${this.params.name}:`, e)
this.updating.resolve()
this.updating = undefined
return
}
this.updating.resolve()
this.updating = undefined
this.data = result
const expiresIn = this.params.expiresIn(result)
this.lastFetchTime = Date.now()
this.expiresAt = this.lastFetchTime + expiresIn
if (!this.params.lazy) {
if (this.timeout) {
clearTimeout(this.timeout)
}
this.timeout = setTimeout(() => {
this.update()
}, expiresIn)
}
}
async get(): Promise<T | null> {
if (this.params.swr && this.data) {
const validator = this.params.swrValidator
if (!validator || validator(this.data, this.expiresAt)) {
this.update().catch(() => {})
return this.data
}
}
await this.update()
return this.data
}
getCached(): T | null {
return this.data
}
}

View file

@ -1,5 +1,3 @@
// include_once(dirname(__FILE__) . "/_secure/umami.php");
import type { APIRoute } from 'astro'
import { umamiLogThisVisit } from '../backend/service/umami'