| Server IP : 167.235.224.122 / Your IP : 216.73.216.110 Web Server : Apache/2.4.58 (Ubuntu) System : Linux newplayground 6.8.0-136-generic #136-Ubuntu SMP PREEMPT_DYNAMIC Wed Jul 1 21:33:11 UTC 2026 aarch64 User : deploy ( 1000) PHP Version : 8.4.23 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : OFF | Sudo : ON | Pkexec : OFF Directory : /var/www/html/openskillpaths/src/ |
Upload File : |
/**
* Credential Engine sync service.
*
* Dedicated module for fetching ceterms:Badge and ceterms:MicroCredential
* records from the Credential Engine Registry and upserting them into the
* local badge_classes table.
*
* CE is intentionally NOT modelled as a node in the nodes table — it has
* its own sync state, its own POST-based paginated API, and its own CTDL
* field mapping that differs structurally from the Badgau/OB3 crawlers.
*
* Environment variables:
* CE_API_KEY — Bearer token from Credential Engine (required to sync)
* CE_SEARCH_URL — Override the default search endpoint (optional)
*
* Docs: https://credreg.net/registry/searchapi
*/
import { db } from './db.js';
// ─── Configuration ─────────────────────────────────────────────────────────────
// TODO: confirm exact endpoint URL with Credential Engine contact
const CE_SEARCH_URL = process.env.CE_SEARCH_URL
?? 'https://credentialengine.org/ce-registry/search';
const CE_API_KEY = process.env.CE_API_KEY ?? '';
// Only these types are in scope — degrees, licenses, apprenticeships etc. are excluded
const CE_TYPES = ['ceterms:Badge', 'ceterms:MicroCredential'];
const PAGE_SIZE = 100;
// ─── Language map helper ───────────────────────────────────────────────────────
// CTDL represents localised strings as language maps: { "en": "...", "fr": "..." }
function langVal(val) {
if (!val) return null;
if (typeof val === 'string') return val;
return val['en'] ?? val['en-US'] ?? Object.values(val)[0] ?? null;
}
// ─── Fetch ─────────────────────────────────────────────────────────────────────
async function fetchCEPage(type, from) {
const body = {
query: { '@type': type },
size: PAGE_SIZE,
from,
};
const headers = {
'Content-Type': 'application/json',
'Accept': 'application/json',
};
if (CE_API_KEY) headers['Authorization'] = `Bearer ${CE_API_KEY}`;
const res = await fetch(CE_SEARCH_URL, {
method: 'POST',
headers,
body: JSON.stringify(body),
signal: AbortSignal.timeout(30_000),
});
if (!res.ok) {
const text = await res.text().catch(() => '');
throw new Error(`CE API HTTP ${res.status}: ${text.slice(0, 300)}`);
}
const json = await res.json();
// TODO: confirm response envelope shape from real API response
// Expected: { data: [...], extra: { total: N } } or { results: [...], total: N }
const items = json?.data ?? json?.results ?? (Array.isArray(json) ? json : []);
const total = json?.extra?.total ?? json?.total ?? null;
return { items, total };
}
async function fetchAllCE(type) {
const results = [];
let from = 0;
while (true) {
const { items, total } = await fetchCEPage(type, from);
results.push(...items);
console.log(`[ce] ${type}: fetched ${results.length}${total != null ? ` / ${total}` : ''}`);
if (items.length < PAGE_SIZE) break;
from += PAGE_SIZE;
}
return results;
}
// ─── CTDL mapper ──────────────────────────────────────────────────────────────
/**
* Map a CTDL JSON-LD credential item to our internal badge_classes row shape.
*
* Key CTDL conventions:
* - Identifiers: @id (URI) or ceterms:ctid (compact form)
* - Localised strings: { "en": "..." } language maps
* - Issuer: ceterms:ownedBy (array of org objects)
* - Skills: ceterms:teaches (array of competency refs)
* - Alignments: ceterms:requires (with nested competency/framework refs)
*
* TODO: refine field mapping once a real API response sample is available.
*/
function mapItemCTDL(item) {
const uuid = item['@id'] ?? item['ceterms:ctid'] ?? null;
if (!uuid) return null;
// ── Issuer ──────────────────────────────────────────────────────────────
// ceterms:ownedBy is an array; take the first entry
const issuerRaw = Array.isArray(item['ceterms:ownedBy'])
? (item['ceterms:ownedBy'][0] ?? {})
: (item['ceterms:ownedBy'] ?? {});
const issuerName = langVal(issuerRaw['ceterms:name'])
?? langVal(issuerRaw['schema:name'])
?? null;
const issuerUrl = issuerRaw['ceterms:subjectWebpage']
?? issuerRaw['schema:url']
?? null;
const issuerId = issuerRaw['@id'] ?? null;
const issuerLogoUrl = issuerRaw['ceterms:image']
?? issuerRaw['schema:image']
?? null;
// ── Skills (ceterms:teaches) ─────────────────────────────────────────────
// Each entry is a competency reference with a URI and label
const teachesRaw = item['ceterms:teaches'] ?? [];
const skills = (Array.isArray(teachesRaw) ? teachesRaw : [teachesRaw])
.map((t) => ({
uri: t['@id'] ?? t['ceasn:ctid'] ?? null,
label: langVal(t['ceasn:competencyText'])
?? langVal(t['ceterms:name'])
?? null,
}))
.filter((s) => s.uri || s.label);
// ── Alignments (ceterms:requires) ────────────────────────────────────────
// TODO: CE's requires structure is nested (condition profiles → competencies)
// and varies by credential type. Expand once real sample is available.
// For now, surface top-level requires entries as alignments where possible.
const requiresRaw = item['ceterms:requires'] ?? [];
const alignments = (Array.isArray(requiresRaw) ? requiresRaw : [requiresRaw])
.filter((r) => r['@id'])
.map((r) => ({
alignmentType: 'requires',
targetName: langVal(r['ceterms:name']) ?? null,
targetUrl: r['@id'] ?? null,
targetCode: r['ceterms:ctid'] ?? null,
educationalFramework: langVal(r['ceterms:inLanguage']) ?? null,
}));
// ── Category ─────────────────────────────────────────────────────────────
// @type may be an array (e.g. ["ceterms:Badge", "ceterms:Credential"])
const type = Array.isArray(item['@type']) ? item['@type'][0] : item['@type'];
return {
uuid,
node_id: null, // CE is not a nodes-table source
name: langVal(item['ceterms:name']) ?? null,
description: langVal(item['ceterms:description']) ?? null,
credential_category: type ?? null,
image_url: item['ceterms:image'] ?? null,
earn_url: item['ceterms:subjectWebpage'] ?? null,
valid_for: null, // no direct CTDL duration equivalent
issuer_name: issuerName,
issuer_did: issuerId,
issuer_url: issuerUrl,
issuer_logo_url: issuerLogoUrl,
skills: JSON.stringify(skills),
alignments: JSON.stringify(alignments),
proof: null, // CE does not use JWS proofs
source_format: 'ctdl',
};
}
// ─── Upsert ───────────────────────────────────────────────────────────────────
async function upsertCEBadgeClass(row) {
await db.query(`
INSERT INTO badge_classes (
uuid, node_id, name, description, credential_category,
image_url, earn_url, valid_for,
issuer_name, issuer_did, issuer_url, issuer_logo_url,
skills, alignments, proof, source_format,
first_seen_at, last_seen_at, updated_at
) VALUES (
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,
NOW(), NOW(), NOW()
)
ON CONFLICT (uuid) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
credential_category = EXCLUDED.credential_category,
image_url = EXCLUDED.image_url,
earn_url = EXCLUDED.earn_url,
issuer_name = EXCLUDED.issuer_name,
issuer_did = EXCLUDED.issuer_did,
issuer_url = EXCLUDED.issuer_url,
issuer_logo_url = EXCLUDED.issuer_logo_url,
skills = EXCLUDED.skills,
alignments = EXCLUDED.alignments,
source_format = EXCLUDED.source_format,
last_seen_at = NOW(),
updated_at = NOW()
`, [
row.uuid, row.node_id, row.name, row.description, row.credential_category,
row.image_url, row.earn_url, row.valid_for,
row.issuer_name, row.issuer_did, row.issuer_url, row.issuer_logo_url,
row.skills, row.alignments, row.proof, row.source_format,
]);
}
// ─── Sync state ───────────────────────────────────────────────────────────────
async function setSyncState(fields) {
const keys = Object.keys(fields);
const values = Object.values(fields);
const set = keys.map((k, i) => `${k} = $${i + 1}`).join(', ');
await db.query(`UPDATE ce_sync_state SET ${set} WHERE id = 1`, values);
}
// ─── Public API ───────────────────────────────────────────────────────────────
/**
* Full sync: fetch all ceterms:Badge and ceterms:MicroCredential from CE
* and upsert into badge_classes.
*/
export async function syncCE() {
if (!CE_API_KEY) {
console.warn('[ce] CE_API_KEY not set — skipping sync');
return { skipped: true, reason: 'CE_API_KEY not configured' };
}
console.log('[ce] starting full sync');
const start = Date.now();
const counts = {};
try {
for (const type of CE_TYPES) {
const items = await fetchAllCE(type);
let count = 0;
for (const item of items) {
const row = mapItemCTDL(item);
if (!row) continue;
await upsertCEBadgeClass(row);
count++;
}
counts[type] = count;
console.log(`[ce] ${type}: upserted ${count}`);
}
await setSyncState({
synced_at: new Date(),
badge_count: counts['ceterms:Badge'] ?? 0,
micro_count: counts['ceterms:MicroCredential'] ?? 0,
last_error: null,
});
console.log(`[ce] sync complete in ${Date.now() - start}ms`, counts);
return { ok: true, counts };
} catch (err) {
console.error('[ce] sync error:', err.message);
await setSyncState({ last_error: err.message }).catch(() => {});
throw err;
}
}
/**
* Return current CE sync state (last run, counts, errors).
*/
export async function getCESyncState() {
const { rows } = await db.query('SELECT * FROM ce_sync_state WHERE id = 1');
return rows[0] ?? null;
}