mirror of
https://github.com/ChenQihan666/Lolia-Nodes-Status-Pages.git
synced 2026-08-14 16:22:34 +08:00
Initial commit
This commit is contained in:
GitHub
@@ -0,0 +1,279 @@
|
||||
import { DurableObject } from 'cloudflare:workers'
|
||||
import { MonitorTarget } from '../../types/config'
|
||||
import { workerConfig } from '../../uptime.config'
|
||||
import { doMonitor, getStatus } from './monitor'
|
||||
import { formatAndNotify, getWorkerLocation } from './util'
|
||||
import { CompactedMonitorStateWrapper, getFromStore, setToStore } from './store'
|
||||
import pLimit from 'p-limit'
|
||||
|
||||
export interface Env {
|
||||
REMOTE_CHECKER_DO: DurableObjectNamespace<RemoteChecker>
|
||||
UPTIMEFLARE_D1: D1Database
|
||||
}
|
||||
|
||||
const Worker = {
|
||||
async scheduled(event: ScheduledEvent, env: Env, ctx: ExecutionContext): Promise<void> {
|
||||
const workerLocation = (await getWorkerLocation()) || 'ERROR'
|
||||
console.log(`Running scheduled event on ${workerLocation}...`)
|
||||
|
||||
// Create a wrapped MonitorState from stored compacted state
|
||||
const state = new CompactedMonitorStateWrapper(await getFromStore(env, 'state'))
|
||||
state.data.overallDown = 0
|
||||
state.data.overallUp = 0
|
||||
|
||||
let statusChanged = false
|
||||
const currentTimeSecond = Math.round(Date.now() / 1000)
|
||||
|
||||
// Parallel check multiple monitors
|
||||
// Max concurrent connection is 6 limited by Cloudflare Workers, we use 5 here to be safe
|
||||
type CheckResult = { id: string; location: string; status: { ping: number; up: boolean; err: string } }
|
||||
let checkQueue: Promise<CheckResult>[] = []
|
||||
let checkResult: Record<string, CheckResult> = {};
|
||||
const limit = pLimit(5);
|
||||
for (const monitor of workerConfig.monitors) {
|
||||
checkQueue.push(limit(() => doMonitor(monitor, workerLocation, env)))
|
||||
}
|
||||
for (const result of await Promise.all(checkQueue)) {
|
||||
checkResult[result.id] = result
|
||||
}
|
||||
|
||||
// Update each monitor's state based on check results
|
||||
for (const monitor of workerConfig.monitors) {
|
||||
console.log(`Processing monitor result: ${monitor.name} (${monitor.id})`)
|
||||
|
||||
let monitorStatusChanged = false
|
||||
const { location: checkLocation, status } = checkResult[monitor.id]
|
||||
|
||||
// Update counters
|
||||
status.up ? state.data.overallUp++ : state.data.overallDown++
|
||||
|
||||
// Update incidents
|
||||
// Create a dummy incident to store the start time of the monitoring and simplify logic
|
||||
if (state.incidentLen(monitor.id) === 0) {
|
||||
state.appendIncident(monitor.id, {
|
||||
start: [currentTimeSecond],
|
||||
end: currentTimeSecond,
|
||||
error: ['dummy'],
|
||||
})
|
||||
}
|
||||
|
||||
// Then lastIncident here must not be null
|
||||
let lastIncident = state.getIncident(monitor.id, state.incidentLen(monitor.id) - 1)
|
||||
|
||||
if (status.up) {
|
||||
// Current status is up
|
||||
// close existing incident if any
|
||||
if (lastIncident.end === null) {
|
||||
lastIncident.end = currentTimeSecond
|
||||
// write back the modified last incident
|
||||
state.setIncident(monitor.id, state.incidentLen(monitor.id) - 1, lastIncident)
|
||||
|
||||
monitorStatusChanged = true
|
||||
try {
|
||||
if (
|
||||
// grace period not set OR ...
|
||||
workerConfig.notification?.gracePeriod === undefined ||
|
||||
// only when we have sent a notification for DOWN status, we will send a notification for UP status (within 30 seconds of possible drift)
|
||||
currentTimeSecond - lastIncident.start[0] >=
|
||||
(workerConfig.notification.gracePeriod + 1) * 60 - 30
|
||||
) {
|
||||
await formatAndNotify(monitor, true, lastIncident.start[0], currentTimeSecond, 'OK')
|
||||
} else {
|
||||
console.log(
|
||||
`grace period (${workerConfig.notification?.gracePeriod}m) not met, skipping webhook UP notification for ${monitor.name}`
|
||||
)
|
||||
}
|
||||
|
||||
console.log('Calling config onStatusChange callback...')
|
||||
await workerConfig.callbacks?.onStatusChange?.(
|
||||
env,
|
||||
monitor,
|
||||
true,
|
||||
lastIncident.start[0],
|
||||
currentTimeSecond,
|
||||
'OK'
|
||||
)
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Current status is down
|
||||
// open new incident if not already open
|
||||
if (lastIncident.end !== null) {
|
||||
state.appendIncident(monitor.id, {
|
||||
start: [currentTimeSecond],
|
||||
end: null,
|
||||
error: [status.err],
|
||||
})
|
||||
monitorStatusChanged = true
|
||||
} else if (lastIncident.end === null && lastIncident.error.slice(-1)[0] !== status.err) {
|
||||
// append if the error message changes
|
||||
lastIncident.start.push(currentTimeSecond)
|
||||
lastIncident.error.push(status.err)
|
||||
|
||||
// write back the modified last incident
|
||||
state.setIncident(monitor.id, state.incidentLen(monitor.id) - 1, lastIncident)
|
||||
monitorStatusChanged = true
|
||||
}
|
||||
|
||||
const currentIncident = state.getIncident(monitor.id, state.incidentLen(monitor.id) - 1)
|
||||
try {
|
||||
if (
|
||||
// monitor status changed AND...
|
||||
(monitorStatusChanged &&
|
||||
// grace period not set OR ...
|
||||
(workerConfig.notification?.gracePeriod === undefined ||
|
||||
// have sent a notification for DOWN status
|
||||
currentTimeSecond - currentIncident.start[0] >=
|
||||
(workerConfig.notification.gracePeriod + 1) * 60 - 30)) ||
|
||||
// grace period is set AND...
|
||||
(workerConfig.notification?.gracePeriod !== undefined &&
|
||||
// grace period is met
|
||||
currentTimeSecond - currentIncident.start[0] >=
|
||||
workerConfig.notification.gracePeriod * 60 - 30 &&
|
||||
currentTimeSecond - currentIncident.start[0] <
|
||||
workerConfig.notification.gracePeriod * 60 + 30)
|
||||
) {
|
||||
if (
|
||||
currentIncident.start[0] !== currentTimeSecond &&
|
||||
workerConfig.notification?.skipErrorChangeNotification
|
||||
) {
|
||||
console.log(
|
||||
'Skipping notification for following error reason change due to user config'
|
||||
)
|
||||
} else {
|
||||
await formatAndNotify(
|
||||
monitor,
|
||||
false,
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`Grace period (${workerConfig.notification
|
||||
?.gracePeriod}m) not met or no change (currently down for ${
|
||||
currentTimeSecond - currentIncident.start[0]
|
||||
}s, changed ${monitorStatusChanged}), skipping webhook DOWN notification for ${
|
||||
monitor.name
|
||||
}`
|
||||
)
|
||||
}
|
||||
|
||||
if (monitorStatusChanged) {
|
||||
console.log('Calling config onStatusChange callback...')
|
||||
await workerConfig.callbacks?.onStatusChange?.(
|
||||
env,
|
||||
monitor,
|
||||
false,
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
|
||||
try {
|
||||
console.log('Calling config onIncident callback...')
|
||||
await workerConfig.callbacks?.onIncident?.(
|
||||
env,
|
||||
monitor,
|
||||
currentIncident.start[0],
|
||||
currentTimeSecond,
|
||||
status.err
|
||||
)
|
||||
} catch (e) {
|
||||
console.log('Error calling callback: ')
|
||||
console.log(e)
|
||||
}
|
||||
}
|
||||
|
||||
// append to latency data
|
||||
state.appendLatency(monitor.id, {
|
||||
loc: checkLocation,
|
||||
ping: status.ping,
|
||||
time: currentTimeSecond,
|
||||
})
|
||||
|
||||
// discard old data
|
||||
while (state.getFirstLatency(monitor.id).time < currentTimeSecond - 12 * 60 * 60) {
|
||||
state.unshiftLatency(monitor.id)
|
||||
}
|
||||
|
||||
// discard old incidents
|
||||
while (
|
||||
state.incidentLen(monitor.id) > 0 &&
|
||||
state.getIncident(monitor.id, 0).end &&
|
||||
state.getIncident(monitor.id, 0).end! < currentTimeSecond - 90 * 24 * 60 * 60
|
||||
) {
|
||||
state.shiftIncident(monitor.id)
|
||||
}
|
||||
|
||||
if (
|
||||
state.incidentLen(monitor.id) === 0 ||
|
||||
(state.getIncident(monitor.id, 0).start[0] > currentTimeSecond - 90 * 24 * 60 * 60 &&
|
||||
state.getIncident(monitor.id, 0).error[0] != 'dummy')
|
||||
) {
|
||||
// put the dummy incident back
|
||||
state.unshiftIncident(monitor.id, {
|
||||
start: [currentTimeSecond - 90 * 24 * 60 * 60],
|
||||
end: currentTimeSecond - 90 * 24 * 60 * 60,
|
||||
error: ['dummy'],
|
||||
})
|
||||
}
|
||||
|
||||
statusChanged ||= monitorStatusChanged
|
||||
}
|
||||
|
||||
console.log(
|
||||
`statusChanged: ${statusChanged}, lastUpdate: ${state.data.lastUpdate}, currentTime: ${currentTimeSecond}`
|
||||
)
|
||||
// Update state
|
||||
// Allow for a cooldown period before writing to storage
|
||||
if (
|
||||
statusChanged ||
|
||||
currentTimeSecond - state.data.lastUpdate >=
|
||||
(workerConfig.kvWriteCooldownMinutes ?? 3) * 60 - 10 // Allow for 10 seconds of clock drift
|
||||
) {
|
||||
console.log('Updating state...')
|
||||
state.data.lastUpdate = currentTimeSecond
|
||||
await setToStore(env, 'state', state.getCompactedStateStr())
|
||||
} else {
|
||||
console.log('Skipping state update due to cooldown period.')
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
export default Worker
|
||||
|
||||
export class RemoteChecker extends DurableObject {
|
||||
constructor(ctx: DurableObjectState, env: Env) {
|
||||
super(ctx, env)
|
||||
}
|
||||
|
||||
async getLocationAndStatus(
|
||||
monitor: MonitorTarget
|
||||
): Promise<{ location: string; status: { ping: number; up: boolean; err: string } }> {
|
||||
const colo = (await getWorkerLocation()) as string
|
||||
console.log(`Running remote checker (DurableObject) at ${colo}...`)
|
||||
const status = await getStatus(monitor)
|
||||
return {
|
||||
location: colo,
|
||||
status: status,
|
||||
}
|
||||
}
|
||||
|
||||
async kill() {
|
||||
// Throwing an error in `blockConcurrencyWhile` will terminate the Durable Object instance
|
||||
// https://developers.cloudflare.com/durable-objects/api/state/#blockconcurrencywhile
|
||||
this.ctx.blockConcurrencyWhile(async () => {
|
||||
throw 'killed'
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
import { Env } from '.'
|
||||
import { MonitorTarget } from '../../types/config'
|
||||
import { withTimeout, fetchTimeout } from './util'
|
||||
|
||||
function isIpAddress(hostname: string): boolean {
|
||||
// `URL.hostname` strips brackets for IPv6, so a `:` reliably indicates an IPv6 literal here.
|
||||
if (hostname.includes(':')) return true
|
||||
|
||||
const parts = hostname.split('.')
|
||||
if (parts.length !== 4) return false
|
||||
|
||||
return parts.every((part) => {
|
||||
if (!/^\d{1,3}$/.test(part)) return false
|
||||
const value = Number(part)
|
||||
return value >= 0 && value <= 255
|
||||
})
|
||||
}
|
||||
|
||||
function getDomainOnlyIpVersionOption(hostname: string, gpUrl: URL): { ipVersion?: number } {
|
||||
// Globalping only allows `measurementOptions.ipVersion` when `target` is a domain (it controls DNS resolution).
|
||||
if (isIpAddress(hostname)) return {}
|
||||
|
||||
// Keep the original behavior for domain targets.
|
||||
return { ipVersion: Number(gpUrl.searchParams.get('ipVersion') || 4) }
|
||||
}
|
||||
|
||||
async function httpResponseBasicCheck(
|
||||
monitor: MonitorTarget,
|
||||
code: number,
|
||||
bodyReader: () => Promise<string>
|
||||
): Promise<string | null> {
|
||||
if (monitor.expectedCodes) {
|
||||
if (!monitor.expectedCodes.includes(code)) {
|
||||
return `Expected codes: ${JSON.stringify(monitor.expectedCodes)}, Got: ${code}`
|
||||
}
|
||||
} else {
|
||||
if (code < 200 || code > 299) {
|
||||
return `Expected codes: 2xx, Got: ${code}`
|
||||
}
|
||||
}
|
||||
|
||||
if (monitor.responseKeyword || monitor.responseForbiddenKeyword) {
|
||||
// Only read response body if we have a keyword to check
|
||||
const responseBody = await bodyReader()
|
||||
|
||||
// MUST contain responseKeyword
|
||||
if (monitor.responseKeyword && !responseBody.includes(monitor.responseKeyword)) {
|
||||
console.log(
|
||||
`${monitor.name} expected keyword ${
|
||||
monitor.responseKeyword
|
||||
}, not found in response (truncated to 100 chars): ${responseBody.slice(0, 100)}`
|
||||
)
|
||||
return "HTTP response doesn't contain the configured keyword"
|
||||
}
|
||||
|
||||
// MUST NOT contain responseForbiddenKeyword
|
||||
if (
|
||||
monitor.responseForbiddenKeyword &&
|
||||
responseBody.includes(monitor.responseForbiddenKeyword)
|
||||
) {
|
||||
console.log(
|
||||
`${monitor.name} forbidden keyword ${
|
||||
monitor.responseForbiddenKeyword
|
||||
}, found in response (truncated to 100 chars): ${responseBody.slice(0, 100)}`
|
||||
)
|
||||
return 'HTTP response contains the configured forbidden keyword'
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export async function getStatusWithGlobalPing(
|
||||
monitor: MonitorTarget
|
||||
): Promise<{ location: string; status: { ping: number; up: boolean; err: string } }> {
|
||||
// TODO: should throw when there's error with globalping API
|
||||
try {
|
||||
if (monitor.checkProxy === undefined) {
|
||||
throw "empty check proxy for globalping, shouldn't call this method"
|
||||
}
|
||||
|
||||
const gpUrl = new URL(monitor.checkProxy)
|
||||
if (gpUrl.protocol !== 'globalping:') {
|
||||
throw 'incorrect check proxy protocol for globalping, got: ' + gpUrl.protocol
|
||||
}
|
||||
|
||||
const token = gpUrl.hostname
|
||||
let globalPingRequest = {}
|
||||
|
||||
if (monitor.method === 'TCP_PING') {
|
||||
const targetUrl = new URL('https://' + monitor.target) // dummy https:// to parse hostname & port
|
||||
const ipVersionOption = getDomainOnlyIpVersionOption(targetUrl.hostname, gpUrl)
|
||||
globalPingRequest = {
|
||||
type: 'ping',
|
||||
target: targetUrl.hostname,
|
||||
locations:
|
||||
gpUrl.searchParams.get('magic') !== null
|
||||
? [
|
||||
{
|
||||
magic: gpUrl.searchParams.get('magic'),
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
measurementOptions: {
|
||||
port: targetUrl.port,
|
||||
packets: 1,
|
||||
protocol: 'tcp', // TODO: icmp?
|
||||
...ipVersionOption,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
const targetUrl = new URL(monitor.target)
|
||||
const ipVersionOption = getDomainOnlyIpVersionOption(targetUrl.hostname, gpUrl)
|
||||
if (monitor.body !== undefined) {
|
||||
throw 'custom body not supported'
|
||||
}
|
||||
if (monitor.method && !['GET', 'HEAD', 'OPTIONS'].includes(monitor.method.toUpperCase())) {
|
||||
throw 'only GET, HEAD, OPTIONS methods are supported'
|
||||
}
|
||||
globalPingRequest = {
|
||||
type: 'http',
|
||||
target: targetUrl.hostname,
|
||||
locations:
|
||||
gpUrl.searchParams.get('magic') !== null
|
||||
? [
|
||||
{
|
||||
magic: gpUrl.searchParams.get('magic'),
|
||||
},
|
||||
]
|
||||
: undefined,
|
||||
measurementOptions: {
|
||||
request: {
|
||||
method: monitor.method,
|
||||
path: targetUrl.pathname,
|
||||
query: targetUrl.search === '' ? undefined : targetUrl.search,
|
||||
headers: Object.fromEntries(
|
||||
Object.entries(monitor.headers ?? {}).map(([key, value]) => [key, String(value)])
|
||||
), // TODO: host header?
|
||||
},
|
||||
port:
|
||||
targetUrl.port === ''
|
||||
? targetUrl.protocol === 'http:'
|
||||
? 80
|
||||
: 443
|
||||
: Number(targetUrl.port),
|
||||
protocol: targetUrl.protocol.replace(':', ''),
|
||||
...ipVersionOption,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const startTime = Date.now()
|
||||
console.log(`Requesting the Global Ping API, payload: ${JSON.stringify(globalPingRequest)}`)
|
||||
const measurement = await fetchTimeout('https://api.globalping.io/v1/measurements', 5000, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: 'Bearer ' + token,
|
||||
},
|
||||
body: JSON.stringify(globalPingRequest),
|
||||
})
|
||||
const measurementResponse = (await measurement.json()) as any
|
||||
|
||||
if (measurement.status !== 202) {
|
||||
throw measurementResponse.error.message
|
||||
}
|
||||
|
||||
const measurementId = measurementResponse.id
|
||||
console.log(
|
||||
`Measurement created successfully, id: ${measurementId}, time elapsed: ${
|
||||
Date.now() - startTime
|
||||
}ms`
|
||||
)
|
||||
|
||||
const pollStart = Date.now()
|
||||
let measurementResult: any
|
||||
while (true) {
|
||||
if (Date.now() - pollStart > (monitor.timeout ?? 10000) + 2000) {
|
||||
// 2s extra buffer
|
||||
throw 'api polling timeout'
|
||||
}
|
||||
|
||||
measurementResult = (await (
|
||||
await fetchTimeout(`https://api.globalping.io/v1/measurements/${measurementId}`, 5000)
|
||||
).json()) as any
|
||||
if (measurementResult.status !== 'in-progress') {
|
||||
break
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Measurement ${measurementId} finished with response: ${JSON.stringify(
|
||||
measurementResult
|
||||
)}, time elapsed: ${Date.now() - pollStart}ms`
|
||||
)
|
||||
|
||||
if (
|
||||
measurementResult.status !== 'finished' ||
|
||||
measurementResult.results[0].result.status !== 'finished'
|
||||
) {
|
||||
console.log(
|
||||
`measurement failed with status: ${measurementResult.status}, result status: ${measurementResult.results[0].result.status}`
|
||||
)
|
||||
// Truncate raw output to avoid huge error messages
|
||||
throw `status [${measurementResult.status}|${
|
||||
measurementResult.results[0].result.status
|
||||
}]: ${measurementResult.results?.[0].result?.rawOutput?.slice(0, 64)}`
|
||||
}
|
||||
|
||||
const country = measurementResult.results[0].probe.country
|
||||
const city = measurementResult.results[0].probe.city
|
||||
|
||||
if (monitor.method === 'TCP_PING') {
|
||||
const time = Math.round(measurementResult.results[0].result.stats.avg)
|
||||
return {
|
||||
location: country + '/' + city,
|
||||
status: {
|
||||
ping: time,
|
||||
up: true,
|
||||
err: '',
|
||||
},
|
||||
}
|
||||
} else {
|
||||
const time = measurementResult.results[0].result.timings.total
|
||||
const code = measurementResult.results[0].result.statusCode
|
||||
const body = measurementResult.results[0].result.rawBody
|
||||
|
||||
let err = await httpResponseBasicCheck(monitor, code, () => body)
|
||||
if (err !== null) {
|
||||
console.log(`${monitor.name} didn't pass response check: ${err}`)
|
||||
}
|
||||
|
||||
if (
|
||||
monitor.target.toLowerCase().startsWith('https') &&
|
||||
!measurementResult.results[0].result.tls.authorized
|
||||
) {
|
||||
console.log(
|
||||
`${monitor.name} TLS certificate not trusted: ${measurementResult.results[0].result.tls.error}`
|
||||
)
|
||||
err = 'TLS certificate not trusted: ' + measurementResult.results[0].result.tls.error
|
||||
}
|
||||
|
||||
return {
|
||||
location: country + '/' + city,
|
||||
status: {
|
||||
ping: time,
|
||||
up: err === null,
|
||||
err: err ?? '',
|
||||
},
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.log(`Globalping ${monitor.name} errored with ${e}`)
|
||||
return {
|
||||
location: 'ERROR',
|
||||
status: {
|
||||
ping: e.toString().toLowerCase().includes('timeout') ? monitor.timeout ?? 10000 : 0,
|
||||
up: false,
|
||||
err: 'Globalping error: ' + e.toString(),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function getStatus(
|
||||
monitor: MonitorTarget
|
||||
): Promise<{ ping: number; up: boolean; err: string }> {
|
||||
let status = {
|
||||
ping: 0,
|
||||
up: false,
|
||||
err: 'Unknown',
|
||||
}
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
if (monitor.method === 'TCP_PING') {
|
||||
// TCP port endpoint monitor
|
||||
try {
|
||||
const connect = await import(/* webpackIgnore: true */ 'cloudflare:sockets').then(
|
||||
(sockets) => sockets.connect
|
||||
)
|
||||
// This is not a real https connection, but we need to add a dummy `https://` to parse the hostname & port
|
||||
const parsed = new URL('https://' + monitor.target)
|
||||
const socket = connect({ hostname: parsed.hostname, port: Number(parsed.port) })
|
||||
|
||||
// Now we have an `opened` promise!
|
||||
await withTimeout(monitor.timeout || 10000, socket.opened)
|
||||
await socket.close()
|
||||
|
||||
console.log(`${monitor.name} connected to ${monitor.target}`)
|
||||
|
||||
status.ping = Date.now() - startTime
|
||||
status.up = true
|
||||
status.err = ''
|
||||
} catch (e: Error | any) {
|
||||
console.log(`${monitor.name} errored with ${e.name}: ${e.message}`)
|
||||
if (e.message.includes('timed out')) {
|
||||
status.ping = monitor.timeout || 10000
|
||||
}
|
||||
status.up = false
|
||||
status.err = e.name + ': ' + e.message
|
||||
}
|
||||
} else {
|
||||
// HTTP endpoint monitor
|
||||
try {
|
||||
let headers = new Headers(monitor.headers as any)
|
||||
if (!headers.has('user-agent')) {
|
||||
headers.set('user-agent', 'UptimeFlare/1.0 (+https://github.com/lyc8503/UptimeFlare)')
|
||||
}
|
||||
|
||||
const response = await fetchTimeout(monitor.target, monitor.timeout || 10000, {
|
||||
method: monitor.method,
|
||||
headers: headers,
|
||||
body: monitor.body,
|
||||
cf: {
|
||||
cacheTtlByStatus: {
|
||||
'100-599': -1, // Don't cache any status code, from https://developers.cloudflare.com/workers/runtime-apis/request/#requestinitcfproperties
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
console.log(`${monitor.name} responded with ${response.status}`)
|
||||
status.ping = Date.now() - startTime
|
||||
|
||||
const err = await httpResponseBasicCheck(
|
||||
monitor,
|
||||
response.status,
|
||||
response.text.bind(response)
|
||||
)
|
||||
try {
|
||||
await response.body?.cancel()
|
||||
} catch (e) {} // Always try to cancel body, see issue #166
|
||||
|
||||
if (err !== null) {
|
||||
console.log(`${monitor.name} didn't pass response check: ${err}`)
|
||||
}
|
||||
status.up = err === null
|
||||
status.err = err ?? ''
|
||||
} catch (e: any) {
|
||||
console.log(`${monitor.name} errored with ${e.name}: ${e.message}`)
|
||||
if (e.name === 'AbortError') {
|
||||
status.ping = monitor.timeout || 10000
|
||||
status.up = false
|
||||
status.err = `Timeout after ${status.ping}ms`
|
||||
} else {
|
||||
status.up = false
|
||||
status.err = e.name + ': ' + e.message
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
|
||||
export async function doMonitor(monitor: MonitorTarget, defaultLocation: string, env: Env) {
|
||||
let checkLocation = defaultLocation
|
||||
let status
|
||||
|
||||
if (monitor.checkProxy) {
|
||||
// Initiate a check using proxy (Geo-specific monitoring)
|
||||
try {
|
||||
console.log(`[${monitor.id}] Calling check proxy: ${monitor.checkProxy}`)
|
||||
let resp
|
||||
if (monitor.checkProxy.startsWith('worker://')) {
|
||||
const doLoc = monitor.checkProxy.replace('worker://', '')
|
||||
const doId = env.REMOTE_CHECKER_DO.idFromName(monitor.id)
|
||||
const doStub = env.REMOTE_CHECKER_DO.get(doId, {
|
||||
locationHint: doLoc as DurableObjectLocationHint,
|
||||
})
|
||||
resp = await doStub.getLocationAndStatus(monitor)
|
||||
try {
|
||||
// Kill the DO instance after use, to avoid extra resource usage
|
||||
await doStub.kill()
|
||||
} catch (err) {
|
||||
// An error here is expected, ignore it
|
||||
}
|
||||
} else if (monitor.checkProxy.startsWith('globalping://')) {
|
||||
resp = await getStatusWithGlobalPing(monitor)
|
||||
} else {
|
||||
resp = await (
|
||||
await fetch(monitor.checkProxy, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(monitor),
|
||||
})
|
||||
).json<{ location: string; status: { ping: number; up: boolean; err: string } }>()
|
||||
}
|
||||
checkLocation = resp.location
|
||||
status = resp.status
|
||||
} catch (err) {
|
||||
console.log(`[${monitor.id}] Error calling proxy: ${err}`)
|
||||
if (monitor.checkProxyFallback) {
|
||||
console.log('Falling back to local check...')
|
||||
status = await getStatus(monitor)
|
||||
} else {
|
||||
// TODO: more consistent error handling (throw or return?)
|
||||
status = { ping: 0, up: false, err: 'Unknown check proxy error' }
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Initiate a check from the current location
|
||||
status = await getStatus(monitor)
|
||||
}
|
||||
|
||||
console.log(`[${monitor.id}] Check result from ${checkLocation}: up=${status.up}, ping=${status.ping}, err=${status.err}`)
|
||||
|
||||
return {
|
||||
location: checkLocation,
|
||||
status,
|
||||
id: monitor.id,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
import { Env } from '.'
|
||||
import {
|
||||
IncidentRecord,
|
||||
LatencyRecord,
|
||||
MonitorState,
|
||||
MonitorStateCompacted,
|
||||
} from '../../types/config'
|
||||
|
||||
export async function getFromStore(env: Env, key: string): Promise<string | null> {
|
||||
const stmt = env.UPTIMEFLARE_D1.prepare('SELECT value FROM uptimeflare WHERE key = ?')
|
||||
const result = await stmt.bind(key).first<{ value: string }>()
|
||||
return result?.value || null
|
||||
}
|
||||
|
||||
export async function setToStore(env: Env, key: string, value: string): Promise<void> {
|
||||
const stmt = env.UPTIMEFLARE_D1.prepare(
|
||||
'INSERT INTO uptimeflare (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value;'
|
||||
)
|
||||
await stmt.bind(key, value).run()
|
||||
}
|
||||
|
||||
export class CompactedMonitorStateWrapper {
|
||||
data: MonitorStateCompacted
|
||||
|
||||
constructor(compactedStateStr: string | null) {
|
||||
if (!compactedStateStr) {
|
||||
// Initialize empty state
|
||||
this.data = {
|
||||
lastUpdate: 0,
|
||||
overallUp: 0,
|
||||
overallDown: 0,
|
||||
incident: {},
|
||||
latency: {},
|
||||
}
|
||||
return
|
||||
}
|
||||
this.data = JSON.parse(compactedStateStr)
|
||||
}
|
||||
|
||||
getCompactedStateStr(): string {
|
||||
return JSON.stringify(this.data)
|
||||
}
|
||||
|
||||
// Don't use this method at server-side
|
||||
uncompact(): MonitorState {
|
||||
let state: MonitorState = {
|
||||
lastUpdate: this.data.lastUpdate,
|
||||
overallUp: this.data.overallUp,
|
||||
overallDown: this.data.overallDown,
|
||||
incident: {},
|
||||
latency: {},
|
||||
}
|
||||
|
||||
const hex2Uint8Arr = (hex: string): Uint8Array => {
|
||||
// @ts-expect-error This method is not available in Node.js 22.x, but available in Cloudflare Workers and new browsers
|
||||
if (Uint8Array.fromHex) {
|
||||
// @ts-expect-error
|
||||
return Uint8Array.fromHex(hex)
|
||||
} else {
|
||||
console.warn('Uint8Array.fromHex is not available, using parseInt as fallback. Consider upgrading your browser.')
|
||||
const ret = new Uint8Array(hex.length / 2)
|
||||
for (let i = 0; i < hex.length; i += 2) {
|
||||
ret[i / 2] = parseInt(hex.slice(i, i + 2), 16)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
}
|
||||
|
||||
Object.keys(this.data.incident).forEach((monitorId) => {
|
||||
state.incident[monitorId] = []
|
||||
const incidents = this.data.incident[monitorId]
|
||||
|
||||
if (
|
||||
incidents.start.length !== incidents.end.length ||
|
||||
incidents.start.length !== incidents.error.length
|
||||
) {
|
||||
throw new Error(
|
||||
'Inconsistent incident data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare'
|
||||
)
|
||||
}
|
||||
|
||||
for (let i = 0; i < incidents.start.length; i++) {
|
||||
state.incident[monitorId].push({
|
||||
start: incidents.start[i],
|
||||
end: incidents.end[i],
|
||||
error: incidents.error[i],
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
Object.keys(this.data.latency).forEach((monitorId) => {
|
||||
state.latency[monitorId] = []
|
||||
const latencies = this.data.latency[monitorId]
|
||||
const locUncompacted: string[] = []
|
||||
latencies.loc.c.forEach((count, index) => {
|
||||
for (let i = 0; i < count; i++) {
|
||||
locUncompacted.push(latencies.loc.v[index])
|
||||
}
|
||||
})
|
||||
|
||||
const timeArr = new Uint32Array(hex2Uint8Arr(latencies.time).buffer)
|
||||
const pingArr = new Uint16Array(hex2Uint8Arr(latencies.ping).buffer)
|
||||
|
||||
if (timeArr.length !== pingArr.length || timeArr.length !== locUncompacted.length) {
|
||||
throw new Error(
|
||||
'Inconsistent latency data lengths, please report an issue at https://github.com/lyc8503/UptimeFlare.'
|
||||
)
|
||||
}
|
||||
|
||||
for (let i = 0; i < timeArr.length; i++) {
|
||||
state.latency[monitorId].push({
|
||||
time: timeArr[i],
|
||||
ping: pingArr[i],
|
||||
loc: locUncompacted[i],
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
incidentLen(monitorId: string): number {
|
||||
const incidents = this.data.incident[monitorId]
|
||||
if (!incidents) return 0
|
||||
return incidents.start.length
|
||||
}
|
||||
|
||||
getIncident(monitorId: string, index: number): IncidentRecord {
|
||||
const incidents = this.data.incident[monitorId]
|
||||
if (!incidents || index < 0 || index >= incidents.start.length) {
|
||||
throw new Error('Index out of bounds or monitor not found')
|
||||
}
|
||||
return {
|
||||
start: incidents.start[index],
|
||||
end: incidents.end[index],
|
||||
error: incidents.error[index],
|
||||
}
|
||||
}
|
||||
|
||||
setIncident(monitorId: string, index: number, incident: IncidentRecord) {
|
||||
const incidents = this.data.incident[monitorId]
|
||||
if (!incidents || index < 0 || index >= incidents.start.length) {
|
||||
throw new Error('Index out of bounds or monitor not found')
|
||||
}
|
||||
incidents.start[index] = incident.start
|
||||
incidents.end[index] = incident.end
|
||||
incidents.error[index] = incident.error
|
||||
}
|
||||
|
||||
appendIncident(monitorId: string, incident: IncidentRecord) {
|
||||
let incidents = this.data.incident[monitorId]
|
||||
if (!incidents) {
|
||||
// Initialize incident arrays
|
||||
this.data.incident[monitorId] = {
|
||||
start: [],
|
||||
end: [],
|
||||
error: [],
|
||||
}
|
||||
incidents = this.data.incident[monitorId]
|
||||
}
|
||||
incidents.start.push(incident.start)
|
||||
incidents.end.push(incident.end)
|
||||
incidents.error.push(incident.error)
|
||||
}
|
||||
|
||||
shiftIncident(monitorId: string) {
|
||||
const incidents = this.data.incident[monitorId]
|
||||
incidents.start.shift()
|
||||
incidents.end.shift()
|
||||
incidents.error.shift()
|
||||
}
|
||||
|
||||
unshiftIncident(monitorId: string, incident: IncidentRecord) {
|
||||
const incidents = this.data.incident[monitorId]
|
||||
incidents.start.unshift(incident.start)
|
||||
incidents.end.unshift(incident.end)
|
||||
incidents.error.unshift(incident.error)
|
||||
}
|
||||
|
||||
latencyLen(monitorId: string): number {
|
||||
const latencies = this.data.latency[monitorId]
|
||||
if (!latencies) return 0
|
||||
return latencies.ping.length / 4 // Uint16Array, 4 characters per entry in hex
|
||||
}
|
||||
|
||||
appendLatency(monitorId: string, record: LatencyRecord) {
|
||||
let latencies = this.data.latency[monitorId]
|
||||
if (!latencies) {
|
||||
// Initialize latency arrays
|
||||
this.data.latency[monitorId] = {
|
||||
time: '',
|
||||
ping: '',
|
||||
loc: {
|
||||
c: [],
|
||||
v: [],
|
||||
},
|
||||
}
|
||||
latencies = this.data.latency[monitorId]
|
||||
}
|
||||
|
||||
// @ts-expect-error
|
||||
latencies.time += new Uint8Array(new Uint32Array([record.time]).buffer).toHex()
|
||||
// @ts-expect-error
|
||||
latencies.ping += new Uint8Array(new Uint16Array([record.ping]).buffer).toHex()
|
||||
|
||||
if (latencies.loc.v[latencies.loc.v.length - 1] !== record.loc) {
|
||||
latencies.loc.c.push(1)
|
||||
latencies.loc.v.push(record.loc)
|
||||
} else {
|
||||
latencies.loc.c[latencies.loc.c.length - 1] += 1
|
||||
}
|
||||
}
|
||||
|
||||
getFirstLatency(monitorId: string): LatencyRecord {
|
||||
let latencies = this.data.latency[monitorId]
|
||||
|
||||
return {
|
||||
// @ts-expect-error
|
||||
time: new Uint32Array(Uint8Array.fromHex(latencies.time.slice(0, 8)).buffer)[0],
|
||||
// @ts-expect-error
|
||||
ping: new Uint16Array(Uint8Array.fromHex(latencies.ping.slice(0, 4)).buffer)[0],
|
||||
loc: latencies.loc.v[0],
|
||||
}
|
||||
}
|
||||
|
||||
getLastLatency(monitorId: string): LatencyRecord {
|
||||
let latencies = this.data.latency[monitorId]
|
||||
|
||||
return {
|
||||
// @ts-expect-error
|
||||
time: new Uint32Array(Uint8Array.fromHex(latencies.time.slice(-8)).buffer)[0],
|
||||
// @ts-expect-error
|
||||
ping: new Uint16Array(Uint8Array.fromHex(latencies.ping.slice(-4)).buffer)[0],
|
||||
loc: latencies.loc.v[latencies.loc.v.length - 1],
|
||||
}
|
||||
}
|
||||
|
||||
unshiftLatency(monitorId: string) {
|
||||
let latencies = this.data.latency[monitorId]
|
||||
|
||||
latencies.time = latencies.time.slice(8)
|
||||
latencies.ping = latencies.ping.slice(4)
|
||||
|
||||
latencies.loc.c[0] -= 1
|
||||
if (latencies.loc.c[0] === 0) {
|
||||
latencies.loc.c.shift()
|
||||
latencies.loc.v.shift()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
import { MonitorTarget, WebhookConfig } from '../../types/config'
|
||||
import { maintenances, workerConfig } from '../../uptime.config'
|
||||
|
||||
async function getWorkerLocation() {
|
||||
const res = await fetch('https://cloudflare.com/cdn-cgi/trace')
|
||||
const text = await res.text()
|
||||
|
||||
const colo = /^colo=(.*)$/m.exec(text)?.[1]
|
||||
return colo
|
||||
}
|
||||
|
||||
const fetchTimeout = (
|
||||
url: string,
|
||||
ms: number,
|
||||
{ signal, ...options }: RequestInit<RequestInitCfProperties> | undefined = {}
|
||||
): Promise<Response> => {
|
||||
const controller = new AbortController()
|
||||
const promise = fetch(url, { signal: controller.signal, ...options })
|
||||
if (signal) signal.addEventListener('abort', () => controller.abort())
|
||||
const timeout = setTimeout(() => controller.abort(), ms)
|
||||
return promise.finally(() => clearTimeout(timeout))
|
||||
}
|
||||
|
||||
function withTimeout<T>(millis: number, promise: Promise<T>): Promise<T> {
|
||||
const timeout = new Promise<T>((resolve, reject) =>
|
||||
setTimeout(() => reject(new Error(`Promise timed out after ${millis}ms`)), millis)
|
||||
)
|
||||
|
||||
return Promise.race([promise, timeout])
|
||||
}
|
||||
|
||||
function formatStatusChangeNotification(
|
||||
monitor: any,
|
||||
isUp: boolean,
|
||||
timeIncidentStart: number,
|
||||
timeNow: number,
|
||||
reason: string,
|
||||
timeZone: string
|
||||
) {
|
||||
const dateFormatter = new Intl.DateTimeFormat('en-US', {
|
||||
month: 'numeric',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
timeZone: timeZone,
|
||||
})
|
||||
|
||||
let downtimeDuration = Math.round((timeNow - timeIncidentStart) / 60)
|
||||
const timeNowFormatted = dateFormatter.format(new Date(timeNow * 1000))
|
||||
const timeIncidentStartFormatted = dateFormatter.format(new Date(timeIncidentStart * 1000))
|
||||
|
||||
if (isUp) {
|
||||
return `✅ ${monitor.name} is up! \nThe service is up again after being down for ${downtimeDuration} minutes.`
|
||||
} else if (timeNow == timeIncidentStart) {
|
||||
return `🔴 ${
|
||||
monitor.name
|
||||
} is currently down. \nService is unavailable at ${timeNowFormatted}. \nIssue: ${
|
||||
reason || 'unspecified'
|
||||
}`
|
||||
} else {
|
||||
return `🔴 ${
|
||||
monitor.name
|
||||
} is still down. \nService is unavailable since ${timeIncidentStartFormatted} (${downtimeDuration} minutes). \nIssue: ${
|
||||
reason || 'unspecified'
|
||||
}`
|
||||
}
|
||||
}
|
||||
|
||||
function templateWebhookPlayload(payload: any, message: string) {
|
||||
for (const key in payload) {
|
||||
if (Object.prototype.hasOwnProperty.call(payload, key)) {
|
||||
if (payload[key] === '$MSG') {
|
||||
payload[key] = message
|
||||
} else if (typeof payload[key] === 'object' && payload[key] !== null) {
|
||||
templateWebhookPlayload(payload[key], message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function webhookNotify(webhook: WebhookConfig, message: string) {
|
||||
if (Array.isArray(webhook)) {
|
||||
for (const w of webhook) {
|
||||
await webhookNotify(w, message)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
console.log(
|
||||
'Sending webhook notification: ' + JSON.stringify(message) + ' to webhook ' + webhook.url
|
||||
)
|
||||
try {
|
||||
let url = webhook.url
|
||||
let method = webhook.method
|
||||
let headers = new Headers(webhook.headers as any)
|
||||
let payloadTemplated: { [key: string]: string | number } = JSON.parse(
|
||||
JSON.stringify(webhook.payload)
|
||||
)
|
||||
templateWebhookPlayload(payloadTemplated, message)
|
||||
let body = undefined
|
||||
|
||||
switch (webhook.payloadType) {
|
||||
case 'param':
|
||||
method = method ?? 'GET'
|
||||
const urlTmp = new URL(url)
|
||||
for (const [k, v] of Object.entries(payloadTemplated)) {
|
||||
urlTmp.searchParams.append(k, v.toString())
|
||||
}
|
||||
url = urlTmp.toString()
|
||||
break
|
||||
case 'json':
|
||||
method = method ?? 'POST'
|
||||
if (headers.get('content-type') === null) {
|
||||
headers.set('content-type', 'application/json')
|
||||
}
|
||||
body = JSON.stringify(payloadTemplated)
|
||||
break
|
||||
case 'x-www-form-urlencoded':
|
||||
method = method ?? 'POST'
|
||||
if (headers.get('content-type') === null) {
|
||||
headers.set('content-type', 'application/x-www-form-urlencoded')
|
||||
}
|
||||
body = new URLSearchParams(payloadTemplated as any).toString()
|
||||
break
|
||||
default:
|
||||
throw 'Unrecognized payload type: ' + webhook.payloadType
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Webhook finalized parameters: ${method} ${url}, headers ${JSON.stringify(
|
||||
Object.fromEntries(headers.entries())
|
||||
)}, body ${JSON.stringify(body)}`
|
||||
)
|
||||
const resp = await fetchTimeout(url, webhook.timeout ?? 5000, { method, headers, body })
|
||||
|
||||
if (!resp.ok) {
|
||||
console.log(
|
||||
'Error calling webhook server, code: ' + resp.status + ', response: ' + (await resp.text())
|
||||
)
|
||||
} else {
|
||||
console.log('Webhook notification sent successfully, code: ' + resp.status)
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error calling webhook server: ' + e)
|
||||
}
|
||||
}
|
||||
|
||||
// Auxiliary function to format notification and send it via webhook
|
||||
const formatAndNotify = async (
|
||||
monitor: MonitorTarget,
|
||||
isUp: boolean,
|
||||
timeIncidentStart: number,
|
||||
timeNow: number,
|
||||
reason: string
|
||||
) => {
|
||||
// Skip notification if monitor is in the skip list
|
||||
const skipList = workerConfig.notification?.skipNotificationIds
|
||||
if (skipList && skipList.includes(monitor.id)) {
|
||||
console.log(`Skipping notification for ${monitor.name} (${monitor.id} in skipNotificationIds)`)
|
||||
return
|
||||
}
|
||||
|
||||
// Skip notification if monitor is in maintenance
|
||||
const maintenanceList = maintenances
|
||||
.filter(
|
||||
(m) =>
|
||||
new Date(timeNow * 1000) >= new Date(m.start) &&
|
||||
(!m.end || new Date(timeNow * 1000) <= new Date(m.end))
|
||||
)
|
||||
.map((e) => e.monitors || [])
|
||||
.flat()
|
||||
|
||||
if (maintenanceList.includes(monitor.id)) {
|
||||
console.log(`Skipping notification for ${monitor.name} (in maintenance)`)
|
||||
return
|
||||
}
|
||||
|
||||
if (workerConfig.notification?.webhook) {
|
||||
const notification = formatStatusChangeNotification(
|
||||
monitor,
|
||||
isUp,
|
||||
timeIncidentStart,
|
||||
timeNow,
|
||||
reason,
|
||||
workerConfig.notification?.timeZone ?? 'Etc/GMT'
|
||||
)
|
||||
await webhookNotify(workerConfig.notification.webhook, notification)
|
||||
} else {
|
||||
console.log(`Webhook not set, skipping notification for ${monitor.name}`)
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
getWorkerLocation,
|
||||
fetchTimeout,
|
||||
withTimeout,
|
||||
webhookNotify,
|
||||
formatStatusChangeNotification,
|
||||
formatAndNotify,
|
||||
}
|
||||
Reference in New Issue
Block a user