Release v2.19.0: indexed content references
This commit is contained in:
lzstealth committed
1 parent
24f1603b43
commit
b59fe08298
31 files changed
+663
-237
No files matched your search
@@ -0,0 +1,198 @@
|
||||
const crypto = require('crypto');
|
||||
|
||||
const REFERENCE_TYPES_BY_FIELD = {
|
||||
source_id: 'api-request',
|
||||
feed_id: 'rss-feed',
|
||||
timetable_group_id: 'timetable-group',
|
||||
weather_location_id: 'weather-location',
|
||||
weather_icon_set_id: 'weather-icon-set'
|
||||
};
|
||||
|
||||
function normalizeMediaReference(value) {
|
||||
const raw = String(value || '').trim();
|
||||
if (!raw) return '';
|
||||
let pathname = raw;
|
||||
if (/^https?:\/\//i.test(raw)) {
|
||||
try { pathname = new URL(raw).pathname; } catch (_error) { return ''; }
|
||||
}
|
||||
pathname = pathname.split(/[?#]/, 1)[0];
|
||||
return pathname.startsWith('/media/') ? pathname : '';
|
||||
}
|
||||
|
||||
function normalizeFontFamily(value) {
|
||||
return String(value || '').trim().replace(/^['"]|['"]$/g, '').replace(/\s+/g, ' ').toLowerCase();
|
||||
}
|
||||
|
||||
function getFontFamilyCandidates(value) {
|
||||
return String(value || '').split(',').map(normalizeFontFamily).filter(Boolean);
|
||||
}
|
||||
|
||||
function addReference(references, seen, ownerKey, referenceType, value) {
|
||||
let referenceValue = String(value === undefined || value === null ? '' : value).trim();
|
||||
if (['api-request', 'rss-feed', 'timetable-group', 'weather-location'].includes(referenceType)) {
|
||||
const numericValue = Number(referenceValue);
|
||||
if (!Number.isFinite(numericValue)) return;
|
||||
referenceValue = String(numericValue);
|
||||
}
|
||||
if (referenceType === 'font-family') referenceValue = normalizeFontFamily(referenceValue);
|
||||
if (!referenceValue) return;
|
||||
const key = String(ownerKey || '') + '\u0000' + referenceType + '\u0000' + referenceValue;
|
||||
if (seen.has(key)) return;
|
||||
seen.add(key);
|
||||
references.push({
|
||||
owner_key: String(ownerKey || ''),
|
||||
reference_type: referenceType,
|
||||
reference_value: referenceValue,
|
||||
reference_hash: crypto.createHash('sha256').update(referenceType + '\u0000' + referenceValue).digest()
|
||||
});
|
||||
}
|
||||
|
||||
function collectValueReferences(value, ownerKey, references, seen) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(function (item) { collectValueReferences(item, ownerKey, references, seen); });
|
||||
return;
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
Object.keys(value).forEach(function (field) {
|
||||
const fieldValue = value[field];
|
||||
const normalizedField = String(field || '').toLowerCase();
|
||||
if (normalizedField === 'font_family' || normalizedField === 'fontfamily') {
|
||||
getFontFamilyCandidates(fieldValue).forEach(function (family) {
|
||||
addReference(references, seen, ownerKey, 'font-family', family);
|
||||
});
|
||||
}
|
||||
const referenceType = REFERENCE_TYPES_BY_FIELD[normalizedField];
|
||||
if (referenceType) addReference(references, seen, ownerKey, referenceType, fieldValue);
|
||||
collectValueReferences(fieldValue, ownerKey, references, seen);
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (typeof value !== 'string') return;
|
||||
|
||||
const embeddedMedia = value.match(/(?:https?:\/\/[^\s"'<>]+)?\/media\/[^\s"'<>)]*/gi) || [];
|
||||
embeddedMedia.forEach(function (mediaValue) {
|
||||
const normalized = normalizeMediaReference(mediaValue);
|
||||
if (normalized) addReference(references, seen, ownerKey, 'media', normalized);
|
||||
});
|
||||
|
||||
const fontFamilies = /font-family\s*:\s*(?:"([^"]+)"|'([^']+)'|([^;}"']+))/gi;
|
||||
let match;
|
||||
while ((match = fontFamilies.exec(value))) {
|
||||
getFontFamilyCandidates(match[1] || match[2] || match[3]).forEach(function (family) {
|
||||
addReference(references, seen, ownerKey, 'font-family', family);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function collectContentReferences(ownerType, ownerId, content) {
|
||||
const references = [];
|
||||
const entries = content && typeof content === 'object' && !Array.isArray(content) ? Object.entries(content) : [];
|
||||
entries.forEach(function (entry) {
|
||||
const ownerKey = String(entry[0] || '').slice(0, 100);
|
||||
const seen = new Set();
|
||||
collectValueReferences(entry[1], ownerKey, references, seen);
|
||||
});
|
||||
return references.map(function (reference) {
|
||||
return Object.assign({ owner_type: ownerType, owner_id: Number(ownerId) }, reference);
|
||||
});
|
||||
}
|
||||
|
||||
async function insertContentReferences(pool, references) {
|
||||
const rows = Array.isArray(references) ? references : [];
|
||||
for (let offset = 0; offset < rows.length; offset += 500) {
|
||||
const batch = rows.slice(offset, offset + 500).map(function (reference) {
|
||||
return [reference.owner_type, reference.owner_id, reference.owner_key, reference.reference_type, reference.reference_value, reference.reference_hash];
|
||||
});
|
||||
if (!batch.length) continue;
|
||||
await pool.query(
|
||||
'INSERT IGNORE INTO c_content_references (owner_type, owner_id, owner_key, reference_type, reference_value, reference_hash) VALUES ?',
|
||||
[batch]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function replaceContentReferences(connection, ownerType, ownerId, content) {
|
||||
const references = collectContentReferences(ownerType, ownerId, content);
|
||||
const [existing] = await connection.query(
|
||||
'SELECT id, owner_key, reference_type, reference_value FROM c_content_references WHERE owner_type = ? AND owner_id = ?',
|
||||
[ownerType, ownerId]
|
||||
);
|
||||
const existingByKey = new Map((existing || []).map(function (reference) {
|
||||
return [JSON.stringify([String(reference.owner_key || ''), String(reference.reference_type || ''), String(reference.reference_value || '')]), reference];
|
||||
}));
|
||||
const nextKeys = new Set(references.map(function (reference) {
|
||||
return JSON.stringify([reference.owner_key, reference.reference_type, reference.reference_value]);
|
||||
}));
|
||||
const staleIds = Array.from(existingByKey.entries())
|
||||
.filter(function (entry) { return !nextKeys.has(entry[0]); })
|
||||
.map(function (entry) { return entry[1].id; });
|
||||
const additions = references.filter(function (reference) {
|
||||
return !existingByKey.has(JSON.stringify([reference.owner_key, reference.reference_type, reference.reference_value]));
|
||||
});
|
||||
|
||||
if (staleIds.length) {
|
||||
await connection.query(
|
||||
'DELETE FROM c_content_references WHERE owner_type = ? AND owner_id = ? AND id IN (?)',
|
||||
[ownerType, ownerId, staleIds]
|
||||
);
|
||||
}
|
||||
await insertContentReferences(connection, additions);
|
||||
}
|
||||
|
||||
async function getContentReferenceValues(pool, referenceType) {
|
||||
const [rows] = await pool.query(
|
||||
'SELECT DISTINCT reference_value FROM c_content_references WHERE reference_type = ?',
|
||||
[referenceType]
|
||||
);
|
||||
return (rows || []).map(function (row) { return String(row.reference_value || ''); }).filter(Boolean);
|
||||
}
|
||||
|
||||
async function getContentReferenceOwnerCounts(pool, referenceType) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT reference_value, COUNT(DISTINCT owner_type, owner_id) AS owner_count
|
||||
FROM c_content_references
|
||||
WHERE reference_type = ?
|
||||
GROUP BY reference_value`,
|
||||
[referenceType]
|
||||
);
|
||||
return new Map((rows || []).map(function (row) {
|
||||
return [String(row.reference_value || ''), Number(row.owner_count) || 0];
|
||||
}));
|
||||
}
|
||||
|
||||
async function getContentReferenceOwners(pool, referenceType) {
|
||||
const [rows] = await pool.query(
|
||||
'SELECT DISTINCT owner_type, owner_id FROM c_content_references WHERE reference_type = ?',
|
||||
[referenceType]
|
||||
);
|
||||
return (rows || []).map(function (row) {
|
||||
return { ownerType: String(row.owner_type || ''), ownerId: Number(row.owner_id) };
|
||||
}).filter(function (owner) {
|
||||
return owner.ownerType && Number.isFinite(owner.ownerId);
|
||||
});
|
||||
}
|
||||
|
||||
async function countContentReferenceOwners(pool, referenceType, referenceValue) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT COUNT(DISTINCT owner_type, owner_id) AS owner_count
|
||||
FROM c_content_references
|
||||
WHERE reference_type = ? AND reference_value = ?`,
|
||||
[referenceType, String(referenceValue || '')]
|
||||
);
|
||||
return Number(rows && rows[0] && rows[0].owner_count) || 0;
|
||||
}
|
||||
|
||||
async function hasContentReference(pool, referenceType, referenceValue) {
|
||||
return (await countContentReferenceOwners(pool, referenceType, referenceValue)) > 0;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectContentReferences,
|
||||
insertContentReferences,
|
||||
replaceContentReferences,
|
||||
getContentReferenceValues,
|
||||
getContentReferenceOwnerCounts,
|
||||
getContentReferenceOwners,
|
||||
countContentReferenceOwners,
|
||||
hasContentReference
|
||||
};
|
||||
Reference in new issue
Block a user