| Server IP : 167.235.224.122 / Your IP : 216.73.216.110 Web Server : Apache/2.4.58 (Ubuntu) System : Linux newplayground 6.8.0-136-generic #136-Ubuntu SMP PREEMPT_DYNAMIC Wed Jul 1 21:33:11 UTC 2026 aarch64 User : deploy ( 1000) PHP Version : 8.4.23 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /var/www/html/openskillpaths/src/ |
Upload File : |
import { db } from './db.js';
const PAGE_SIZE = 100;
/**
* Fetch all badge classes from a node's API endpoint, paginating until done.
*/
async function fetchAll(apiEndpoint) {
const results = [];
let offset = 0;
while (true) {
const url = `${apiEndpoint}?limit=${PAGE_SIZE}&offset=${offset}`;
const res = await fetch(url, {
headers: { 'Accept': 'application/json' },
signal: AbortSignal.timeout(15_000),
});
if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
const body = await res.json();
// Support both { data: [...] } and plain array responses
const items = Array.isArray(body) ? body : (body.data ?? []);
results.push(...items);
if (items.length < PAGE_SIZE) break;
offset += PAGE_SIZE;
}
return results;
}
/**
* Map a Badgau schema.org EducationalOccupationalCredential item to our DB row shape.
*/
function mapItemBadgau(item, nodeId) {
const issuer = item.recognizedBy ?? {};
const skills = (item.competencyRequired ?? []).map((s) => ({
uri: s.termCode ?? null,
label: s.name ?? null,
}));
const alignments = (item.educationalAlignment ?? []).map((a) => ({
alignmentType: a.alignmentType ?? null,
targetName: a.targetName ?? null,
targetUrl: a.targetUrl ?? null,
targetCode: a.targetCode ?? null,
educationalFramework: a.educationalFramework ?? null,
}));
return {
uuid: item.identifier,
node_id: nodeId,
name: item.name ?? null,
description: item.description ?? null,
credential_category: item.credentialCategory ?? null,
image_url: item.image ?? null,
earn_url: item.url ?? null,
valid_for: item.validFor ?? null,
issuer_name: issuer.name ?? null,
issuer_did: issuer.identifier ?? null,
issuer_url: issuer.url ?? null,
issuer_logo_url: issuer.logo ?? null,
skills: skills,
alignments: alignments,
proof: item.proof ?? null,
source_format: 'badgau',
};
}
/**
* Map an Open Badges 3.0 Achievement item to our DB row shape.
*
* OB3 spec: https://www.imsglobal.org/spec/ob/v3p0/
* Key differences from Badgau:
* - identifier is 'id' (a URI), not 'identifier'
* - issuer is 'issuer' object, not 'recognizedBy'
* - skills come from 'tag' (strings), not 'competencyRequired'
* - alignments come from 'alignment', not 'educationalAlignment'
* - category is 'achievementType', not 'credentialCategory'
* - image may be { id: url } or a plain string
*/
function mapItemOB3(item, nodeId) {
const issuer = item.issuer ?? {};
// OB3 tags are plain strings — store as skills with label only
const tagSkills = (item.tag ?? []).map((tag) => ({
uri: null,
label: String(tag),
}));
// Competency-typed alignments are also surfaced as skills for searchability
const COMPETENCY_TYPES = ['ceasn:Competency', 'CTDL_competency', 'esco:Occupation', 'esco:Skill'];
const alignmentSkills = (item.alignment ?? [])
.filter((a) => COMPETENCY_TYPES.includes(a.targetType))
.map((a) => ({
uri: a.targetUrl ?? a.targetCode ?? null,
label: a.targetName ?? null,
}));
const skills = [...tagSkills, ...alignmentSkills];
const alignments = (item.alignment ?? []).map((a) => ({
alignmentType: a.targetType ?? null,
targetName: a.targetName ?? null,
targetUrl: a.targetUrl ?? null,
targetCode: a.targetCode ?? null,
educationalFramework: a.targetFramework ?? null,
}));
// image can be { id: url } or a plain string
const imageUrl = typeof item.image === 'object'
? (item.image?.id ?? null)
: (item.image ?? null);
// issuer logo can be { id: url } or a plain string
const issuerLogo = typeof issuer.image === 'object'
? (issuer.image?.id ?? null)
: (issuer.image ?? null);
return {
uuid: item.id,
node_id: nodeId,
name: item.name ?? null,
description: item.description ?? null,
credential_category: item.achievementType ?? null,
image_url: imageUrl,
earn_url: item.criteria?.id ?? null,
valid_for: null, // no OB3 equivalent
issuer_name: issuer.name ?? null,
issuer_did: issuer.id ?? null,
issuer_url: issuer.url ?? null,
issuer_logo_url: issuerLogo,
skills: skills,
alignments: alignments,
proof: item.proof ?? null,
source_format: 'ob3',
};
}
/**
* Dispatch to the correct mapper based on node format.
*/
function mapItem(item, nodeId, format) {
if (format === 'ob3') return mapItemOB3(item, nodeId);
return mapItemBadgau(item, nodeId);
}
/**
* Upsert a single badge class record.
*/
async function upsertBadgeClass(row) {
await db.query(`
INSERT INTO badge_classes (
uuid, node_id, name, description, credential_category,
image_url, earn_url, valid_for,
issuer_name, issuer_did, issuer_url, issuer_logo_url,
skills, alignments, proof, source_format,
status, first_seen_at, last_seen_at, updated_at
) VALUES (
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,
'active', NOW(), NOW(), NOW()
)
ON CONFLICT (uuid) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
credential_category = EXCLUDED.credential_category,
image_url = EXCLUDED.image_url,
earn_url = EXCLUDED.earn_url,
valid_for = EXCLUDED.valid_for,
issuer_name = EXCLUDED.issuer_name,
issuer_did = EXCLUDED.issuer_did,
issuer_url = EXCLUDED.issuer_url,
issuer_logo_url = EXCLUDED.issuer_logo_url,
skills = EXCLUDED.skills,
alignments = EXCLUDED.alignments,
proof = EXCLUDED.proof,
source_format = EXCLUDED.source_format,
status = 'active',
last_seen_at = NOW(),
updated_at = NOW()
`, [
row.uuid, row.node_id, row.name, row.description, row.credential_category,
row.image_url, row.earn_url, row.valid_for,
row.issuer_name, row.issuer_did, row.issuer_url, row.issuer_logo_url,
row.skills, row.alignments, row.proof, row.source_format,
]);
}
/**
* Crawl a single node. Updates node status/timestamps after.
*/
export async function crawlNode(node) {
console.log(`[crawler] crawling node #${node.id}: ${node.url} (${node.node_type ?? 'badgau'})`);
const start = Date.now();
try {
const items = await fetchAll(node.api_endpoint);
let count = 0;
const seenUuids = [];
for (const item of items) {
// Badgau uses 'identifier', OB3 uses 'id'
if (!item.identifier && !item.id) continue;
const mapped = mapItem(item, node.id, node.node_type ?? 'badgau');
seenUuids.push(mapped.uuid);
await upsertBadgeClass(mapped);
count++;
}
// Archive badge classes from this node that were not in the latest crawl
if (seenUuids.length > 0) {
const { rowCount: archived } = await db.query(`
UPDATE badge_classes
SET status = 'archived', updated_at = NOW()
WHERE node_id = $1
AND status = 'active'
AND uuid != ALL($2)
`, [node.id, seenUuids]);
if (archived > 0) {
console.log(`[crawler] node #${node.id}: archived ${archived} badge classes no longer in source`);
}
}
await db.query(`
UPDATE nodes SET
last_crawled_at = NOW(),
last_error = NULL,
status = 'active',
updated_at = NOW()
WHERE id = $1
`, [node.id]);
console.log(`[crawler] node #${node.id} done — ${count} badge classes (${Date.now() - start}ms)`);
} catch (err) {
console.error(`[crawler] node #${node.id} error: ${err.message}`);
await db.query(`
UPDATE nodes SET
last_error = $1,
status = 'error',
updated_at = NOW()
WHERE id = $2
`, [err.message, node.id]);
}
}
/**
* Crawl all nodes that are due (last_crawled_at is NULL or older than crawl_interval).
*/
export async function crawlDue() {
const { rows: nodes } = await db.query(`
SELECT * FROM nodes
WHERE status != 'paused'
AND (
last_crawled_at IS NULL
OR last_crawled_at < NOW() - INTERVAL '1 second' * crawl_interval
)
`);
if (nodes.length === 0) {
console.log('[crawler] no nodes due for crawl');
return;
}
// Crawl sequentially to keep DB load predictable
for (const node of nodes) {
await crawlNode(node);
}
}