import {join} from "path" import {readFile, writeFile} from "fs/promises" import { asDate, asNumber, assertField, assertNoExtraFields, asString, isArray, isObject, isString, } from "./typeassert"; import {DateTime, Duration, IANAZone, Zone} from "luxon"; import {default as escapeHTML} from "escape-html"; interface UserMessages { readonly group: readonly User[] readonly messages: readonly Message[] } function groupToCompanionNames(group: readonly User[], localEmail: string): readonly string[] { return group.filter((user) => user.email !== localEmail).map((user) => user.name) } async function handleUserDir(path: string): Promise { const messagesPath = join(path, "messages.json") const messages = readFile(messagesPath, {encoding: "utf-8"}) .then((messagesText) => JSON.parse(messagesText)) .then((messagesJson) => handleMessageFile(messagesJson, messagesPath)) const groupPath = join(path, "group_info.json") const group = readFile(groupPath, {encoding: "utf-8"}) .then((groupText) => JSON.parse(groupText)) .then((groupJson) => handleGroupFile(groupJson, groupPath)) return { group: await group, messages: await messages, } } const groupFileExpectedFields = new Set(["members"]) async function handleGroupFile(groupJson: unknown, groupFilePath: string): Promise { if (!(isObject(groupJson, groupFilePath) && assertField(groupJson, "members", groupFilePath))) { return [] } assertNoExtraFields(groupJson, groupFileExpectedFields) const context = `${groupFilePath} .members` if (!isArray(groupJson.members, context)) { return [] } const result: User[] = [] let index = -1; for (const memberJson of groupJson.members) { index += 1 const member = await handleUser(memberJson, `${context}[${index}]`) if (member) { result.push(member) } } return result } interface Message { readonly topic: string readonly text?: string readonly createdDate: DateTime readonly creator: User readonly annotations: readonly Annotation[] readonly attachedFiles: readonly AttachedFile[] } const messageFileExpectedFields = new Set(["messages"]) async function handleMessageFile(messagesJson: unknown, messageFilePath: string): Promise { if (!(isObject(messagesJson, messageFilePath) && assertField(messagesJson, "messages", messageFilePath))) { return [] } assertNoExtraFields(messagesJson, messageFileExpectedFields) return handleMessageList(messagesJson.messages, messageFilePath) } async function handleMessageList(messagesJson: unknown, messageFile: string): Promise { const context = `${messageFile} .messages` if (!isArray(messagesJson, context)) { return [] } const result: Message[] = [] let index = -1; for (const messageJson of messagesJson) { index += 1 const message = await handleMessage(messageJson, `${context}[${index}]`) if (message) { result.push(message) } } return result } const messageExpectedFields = new Set(["text", "created_date", "topic_id", "creator", "annotations", "attached_files"]) async function handleMessage(messageJson: unknown, context: string): Promise { if (!isObject(messageJson, context)) { return null } assertNoExtraFields(messageJson, messageExpectedFields, context) const text = "text" in messageJson ? asString(messageJson.text, context + ".text") : undefined const createdDate = assertField(messageJson, "created_date", context) ? asDate(messageJson.created_date, context + ".created_date") : null const topic = assertField(messageJson, "topic_id", context) ? asString(messageJson.topic_id, context + ".topic_id") : null const creator = assertField(messageJson, "creator", context) ? await handleUser(messageJson.creator, context + ".creator") : null const annotations = "annotations" in messageJson ? await handleAnnotationList(messageJson.annotations, context + ".annotations") : [] const attachedFiles = "attached_files" in messageJson ? await handleAttachedFilesList(messageJson.attached_files, context + ".attachedFiles") : [] if (text === null || createdDate === null || topic === null || creator === null) { return null } return { text, createdDate, topic, creator, annotations, attachedFiles } } interface User { readonly name: string readonly email: string readonly userType: string } const creatorExpectedFields = new Set(["name", "email", "user_type"]) async function handleUser(creatorJson: unknown, context: string): Promise { if (!isObject(creatorJson, context)) { return null } assertNoExtraFields(creatorJson, creatorExpectedFields, context) const name = assertField(creatorJson, "name", context) ? asString(creatorJson.name, context + ".name") : null const email = assertField(creatorJson, "email", context) ? asString(creatorJson.email, context + ".email") : null const userType = assertField(creatorJson, "user_type", context) ? asString(creatorJson.user_type, context + ".user_type") : null if (name === null || email === null || userType === null) { return null } return { name, email, userType } } interface BaseAnnotation { readonly type: string readonly start: number readonly length: number } interface FormatAnnotation extends BaseAnnotation { readonly type: "format" readonly format: FormatType } interface URLAnnotation extends BaseAnnotation { readonly type: "url" readonly url: string } type Annotation = FormatAnnotation | URLAnnotation async function handleAnnotationList(annotationsJson: unknown, context: string): Promise { if (!isArray(annotationsJson, context)) { return [] } const result: Annotation[] = [] let index = -1; for (const annotationJson of annotationsJson) { index += 1 const annotation = await handleAnnotation(annotationJson, `${context}[${index}]`) if (annotation) { result.push(annotation) } } return result } const annotationExpectedFields = new Set(["start_index", "length", "format_metadata", "youtube_metadata", "url_metadata", "drive_metadata"]) async function handleAnnotation(annotationJson: unknown, context: string): Promise { if (!isObject(annotationJson, context)) { return null } assertNoExtraFields(annotationJson, annotationExpectedFields, context) const start = assertField(annotationJson, "start_index", context) ? asNumber(annotationJson.start_index, context + ".start_index") : null const length = assertField(annotationJson, "length", context) ? asNumber(annotationJson.length, context + ".length") : null if (start === null || length === null) { return null } const types = new Set<"format"|"youtube"|"url"|"drive">() let format: FormatType|null = null if ("format_metadata" in annotationJson) { types.add("format") format = await handleFormatMetadata(annotationJson.format_metadata, context + ".format_metadata") } let url: string|null = null if ("youtube_metadata" in annotationJson) { types.add("youtube") url = await handleYoutubeMetadata(annotationJson.youtube_metadata, context + ".youtube_metadata") } if ("url_metadata" in annotationJson) { types.add("url") url = await handleURLMetadata(annotationJson.url_metadata, context + ".url_metadata") } if ("drive_metadata" in annotationJson) { types.add("drive") url = await handleDriveMetadata(annotationJson.drive_metadata, context + ".drive_metadata") } if (types.size > 1) { console.log(`Too many metadata - ${Array.from(types).join(", ")} (in ${context})`) } else if (types.size === 0) { console.log(`No metadata (in ${context})`) } if (format !== null) { return { type: "format", start, length, format, } } else if (url !== null) { return { type: "url", start, length, url, } } else { console.log(`Failed to parse metadata for ${Array.from(types).join(", ")} (in ${context})`) return null } } enum FormatType { BOLD = "BOLD", ITALIC = "ITALIC", STRIKE = "STRIKE", } const formatMetadataExpectedFields = new Set(["format_type"]) async function handleFormatMetadata(formatMetadataJson: unknown, context: string): Promise { if (!isObject(formatMetadataJson, context)) { return null } assertNoExtraFields(formatMetadataJson, formatMetadataExpectedFields, context) if (!assertField(formatMetadataJson, "format_type", context) || !isString(formatMetadataJson.format_type, context + ".format_type")) { return null } switch (formatMetadataJson.format_type) { case FormatType.BOLD: return FormatType.BOLD case FormatType.ITALIC: return FormatType.ITALIC case FormatType.STRIKE: return FormatType.STRIKE default: console.log(`unknown format type ${formatMetadataJson.format_type}`) return null } } const youtubeMetadataExpectedFields = new Set(["id", "start_time"]) async function handleYoutubeMetadata(youtubeMetadataJson: unknown, context: string): Promise { if (!isObject(youtubeMetadataJson, context)) { return null } assertNoExtraFields(youtubeMetadataJson, youtubeMetadataExpectedFields, context) const id = assertField(youtubeMetadataJson, "id", context) ? asString(youtubeMetadataJson.id, context + ".id") : null const startTime = assertField(youtubeMetadataJson, "start_time", context) ? asNumber(youtubeMetadataJson.start_time, context + ".start_time") : null if (id === null || startTime === null) { return null } return `https://youtube.com/watch?v=${id}&t=${startTime}s` } const urlMetadataExpectedFields = new Set(["title", "snippet", "image_url", "url"]) async function handleURLMetadata(urlMetadataJson: unknown, context: string): Promise { if (!isObject(urlMetadataJson, context)) { return null } assertNoExtraFields(urlMetadataJson, urlMetadataExpectedFields, context) const url = assertField(urlMetadataJson, "url", context) && isObject(urlMetadataJson.url) && assertField(urlMetadataJson.url, "private_do_not_access_or_else_safe_url_wrapped_value", context + ".url") ? asString(urlMetadataJson.url.private_do_not_access_or_else_safe_url_wrapped_value, context + ".url") : null if (url === null) { return null } return url } const driveMetadataExpectedFields = new Set(["id", "title", "thumbnail_url"]) async function handleDriveMetadata(driveMetadataJson: unknown, context: string): Promise { if (!isObject(driveMetadataJson, context)) { return null } const id = assertField(driveMetadataJson, "id", context) ? asString(driveMetadataJson.id, context + ".id") : null assertNoExtraFields(driveMetadataJson, driveMetadataExpectedFields, context) if (id === null) { return null } return `https://drive.google.com/file/d/${id}/view` } interface AttachedFile { originalName: string exportName: string } async function handleAttachedFilesList(attachedFilesJson: unknown, context: string): Promise { if (!isArray(attachedFilesJson, context)) { return [] } const result: AttachedFile[] = [] let index = -1; for (const attachedFileJson of attachedFilesJson) { index += 1 const attachedFile = await handleAttachedFile(attachedFileJson, `${context}[${index}]`) if (attachedFile) { result.push(attachedFile) } } return result } const attachedFileExpectedFields = new Set(["original_name", "export_name"]) async function handleAttachedFile(attachedFileJson: unknown, context: string): Promise { if (!isObject(attachedFileJson, context)) { return null } assertNoExtraFields(attachedFileJson, attachedFileExpectedFields, context) const originalName = assertField(attachedFileJson, "original_name", context) ? asString(attachedFileJson.original_name, context + ".original_name") : null const exportName = assertField(attachedFileJson, "export_name", context) ? asString(attachedFileJson.export_name, context + ".export_name") : null if (originalName === null || exportName === null) { return null } return { originalName, exportName, } } const DefaultMessageZone = IANAZone.create("UTC") const DefaultMessageThreshold = Duration.fromObject({hours: 4}) async function divideMessages(messages: readonly Message[], threshold?: Duration, targetTimezone?: Zone): Promise { const results: (readonly Message[])[] = [] if (messages.length === 0) { return results } let startPoint = 0 let currentPoint = 1 let startDate = messages[0].createdDate.setZone(targetTimezone ?? DefaultMessageZone) let previousDateAdjusted: DateTime = startDate.plus(threshold ?? DefaultMessageThreshold) for (; currentPoint < messages.length; currentPoint += 1) { const currentMessage = messages[currentPoint] const currentDate = currentMessage.createdDate.setZone(targetTimezone ?? DefaultMessageZone) if (startPoint < currentPoint && !startDate.hasSame(currentDate, "day") && (previousDateAdjusted <= currentDate)) { results.push(messages.slice(startPoint, currentPoint)) startPoint = currentPoint startDate = currentDate } previousDateAdjusted = currentDate.plus(threshold ?? DefaultMessageThreshold) } results.push(messages.slice(startPoint)) return results } interface Interruption { offset: number interruption: string|{annotation: Annotation, index: number, start: boolean, end: boolean} } const MessageContentInterruptions = /\n\n|\n/ async function renderMessageText(text: string, annotations: readonly Annotation[]): Promise { let remainingText = text let remainingAnnotations: (Annotation & {index: number, started: boolean, ended: boolean})[] = annotations.slice() .map((annotation, index) => ({...annotation, index, started: false, ended: false})) let position = 0; const result = ['

'] const stack: string[] = [] let nextInterruption: Interruption|null = null function findNextInterruption(): Interruption|null { const nextEscape = MessageContentInterruptions.exec(remainingText) let upcomingInterruption: Interruption|null = nextEscape ? {offset: nextEscape.index, interruption: nextEscape[0]} : null for (const annotation of remainingAnnotations) { const nextOffset = upcomingInterruption?.offset ?? Number.POSITIVE_INFINITY const nextInterruption = upcomingInterruption?.interruption const nextAnno = nextInterruption && typeof nextInterruption !== "string" && "annotation" in nextInterruption ? nextInterruption : null const effectiveStart = annotation.start - position const effectiveEnd = effectiveStart + annotation.length if (!annotation.started) { if (effectiveStart < nextOffset || (effectiveStart === nextOffset && annotation.index < (nextAnno?.index ?? Number.POSITIVE_INFINITY))) { upcomingInterruption = { offset: effectiveStart, interruption: { annotation, index: annotation.index, start: true, end: (effectiveStart === effectiveEnd) } } } } else if (!annotation.ended) { if (effectiveEnd < nextOffset || (effectiveEnd === nextOffset && annotation.index > (nextAnno?.index ?? Number.NEGATIVE_INFINITY))) { upcomingInterruption = { offset: effectiveEnd, interruption: { annotation, index: annotation.index, start: false, end: true } } } } else { console.log(`already exhausted annotation ${JSON.stringify(annotation)}`) } } return upcomingInterruption } while ((nextInterruption = findNextInterruption())) { let offset = nextInterruption.offset result.push(escapeHTML(remainingText.slice(0, offset))) if (typeof nextInterruption.interruption === "string") { switch (nextInterruption.interruption) { case "\n\n": result.push("

") break; case "\n": result.push("
") break; default: console.log(`unknown string interruption ${JSON.stringify(nextInterruption.interruption)}`) break; } offset += nextInterruption.interruption.length } else { switch (nextInterruption.interruption.annotation.type) { case "url": if (nextInterruption.interruption.start) { result.push(``) stack.push("") } if (nextInterruption.interruption.end) { if (stack.pop() !== "") { console.log("bad stacking") } result.push("") } break case "format": switch (nextInterruption.interruption.annotation.format) { case FormatType.BOLD: if (nextInterruption.interruption.start) { result.push("") stack.push("") } if (nextInterruption.interruption.end) { if (stack.pop() !== "") { console.log("bad stacking") } result.push("") } break case FormatType.ITALIC: if (nextInterruption.interruption.start) { result.push("") stack.push("") } if (nextInterruption.interruption.end) { if (stack.pop() !== "") { console.log("bad stacking") } result.push("") } break case FormatType.STRIKE: if (nextInterruption.interruption.start) { result.push("") stack.push("") } if (nextInterruption.interruption.end) { if (stack.pop() !== "") { console.log("bad stacking") } result.push("") } break default: console.log(`unknown format type ${nextInterruption.interruption.annotation.format}`) break } break } const idx = nextInterruption.interruption.index const match = remainingAnnotations.find((anno) => anno.index === idx) if (match && nextInterruption.interruption.start) { match.started = true } if (match && nextInterruption.interruption.end) { match.ended = true remainingAnnotations.splice(remainingAnnotations.findIndex((anno) => anno.index === idx), 1) } } remainingText = remainingText.slice(offset) position += offset } result.push(escapeHTML(remainingText)) result.push("

") return result.join("") } async function renderMessageAttachments(attachments: readonly AttachedFile[]): Promise { const result: string[] = [] for (const attachment of attachments) { result.push(`${attachment.originalName}`) } return result.join("") } async function renderMessageList(messages: readonly Message[], {threshold, localEmail, displayZone}: {threshold: Duration, localEmail: string, displayZone: Zone}): Promise { const result: string[] = [] let lastDate: DateTime|null = null let lastAuthor: string|null = null for (const message of messages) { const nextDate = message.createdDate.setZone(displayZone) const needsDatestamp = lastDate === null || !nextDate.hasSame(lastDate, "day") const needsTimestamp = lastDate === null || nextDate >= lastDate.plus(threshold) if (needsDatestamp || needsTimestamp) { result.push(`

${ needsDatestamp ? escapeHTML(nextDate.toLocaleString(DateTime.DATE_FULL)) : ""}${ needsDatestamp && needsTimestamp ? "
" : ""}${ needsTimestamp ? escapeHTML(nextDate.toLocaleString(DateTime.TIME_SIMPLE)) : ""}

`) } const nextAuthor = message.creator.email const isSelf = nextAuthor === localEmail if (lastAuthor !== nextAuthor) { result.push(`

${escapeHTML(message.creator.name)}

`) } result.push(`
${message.text ? await renderMessageText(message.text, message.annotations) : ""} ${message.attachedFiles ? await renderMessageAttachments(message.attachedFiles) : ""}
`) lastDate = nextDate lastAuthor = nextAuthor } return result } async function renderMessageLists(dividedMessages: readonly (readonly Message[])[], {group, threshold, localEmail, displayZone, destination}: {group: readonly User[], threshold: Duration, localEmail: string, displayZone: Zone, destination: string}): Promise { for (const messages of dividedMessages) { const last = messages[messages.length - 1] const first = messages[0] const lastDate = last.createdDate.setZone(displayZone) const firstDate = first.createdDate.setZone(displayZone) const basename = firstDate.toISO({includeOffset: false, format: "basic", suppressMilliseconds: true, suppressSeconds: false, includePrefix: true}) const title = `Chat with ${groupToCompanionNames(group, localEmail).join(", ")} from ${firstDate.toLocaleString(DateTime.DATETIME_FULL)} (${localEmail})` const head = `Chat with ${groupToCompanionNames(group, localEmail).join(", ")}` const subhead = `From ${firstDate.toLocaleString(DateTime.DATETIME_FULL)} to ${lastDate.toLocaleString(DateTime.DATETIME_FULL)} (${localEmail})` const messageElements = (await renderMessageList(messages, {threshold, localEmail, displayZone})).join("\n") const html = ` ${escapeHTML(title)}

${escapeHTML(head)}

${escapeHTML(subhead)}

Theme: / /

${messageElements}
` await writeFile(join(destination, basename + ".html"), html, {encoding: 'utf-8'}) } } async function main(directory: string, localEmail?: string, zoneName?: string, destination?: string) { const zone = zoneName ? IANAZone.create(zoneName) : DateTime.local().zone const contents = await handleUserDir(directory ?? ".") const dividedMessagesReya = await divideMessages(contents.messages, Duration.fromObject({hours: 4}), zone) await renderMessageLists(dividedMessagesReya, { group: contents.group, threshold: Duration.fromObject({hours: 1}), localEmail: localEmail ?? "", displayZone: zone, destination: destination ?? directory ?? "."}) } const [directory, localEmail, zoneName, destination] = process.argv.slice(2) main(directory, localEmail, zoneName, destination).then(() => console.log("done")).catch((ex) => console.log(ex))