roost.tools
Mirrored from GitHub
github.com/roostorg/coop
81 kB
2562 lines
1/* eslint-disable max-lines */
2import type { Exception } from '@opentelemetry/api';
3import { makeEnumLike, type ItemIdentifier } from '@roostorg/coop-types';
4import _Ajv from 'ajv';
5import { sql, type Kysely } from 'kysely';
6import _ from 'lodash';
7import { FormData } from 'undici';
8import { js2xml } from 'xml-js';
9
10import { type Dependencies } from '../../iocContainer/index.js';
11import { jsonStringify } from '../../utils/encoding.js';
12import { type JSONSchemaV4 } from '../../utils/json-schema-types.js';
13import { type FixKyselyRowCorrelation } from '../../utils/kysely.js';
14import { logErrorJson } from '../../utils/logging.js';
15import { assertUnreachable, withRetries } from '../../utils/misc.js';
16import {
17 type CollapseCases,
18 type NonEmptyArray,
19} from '../../utils/typescript-types.js';
20import { rawItemSubmissionToItemSubmission } from '../itemProcessingService/makeItemSubmission.js';
21import { type RawItemData } from '../itemProcessingService/toNormalizedItemDataOrErrors.js';
22import {
23 rawItemSubmissionSchema,
24 type RawItemSubmission,
25} from '../itemProcessingService/types.js';
26import type {
27 NCMECReportedContentInThread,
28 NCMECThreadReport,
29} from '../manualReviewToolService/modules/JobDecisioning.js';
30import {
31 makeFormDataLikeWithStreams,
32 type FormDataLikeWithStreams,
33} from '../networkingService/index.js';
34import { type NcmecReportingServicePg } from './dbTypes.js';
35import {
36 ncmecDebugDump,
37 ncmecDebugEnabled,
38 ncmecDebugLog,
39} from './ncmecDebug.js';
40import { summarizeNcmecErrorForReviewer } from './ncmecReviewerErrors.js';
41
42export const NCMECEvent = makeEnumLike([
43 'Login',
44 'Registration',
45 'Purchase',
46 'Upload',
47 'Other',
48 'Unknown',
49]);
50export type NCMECEventType = keyof typeof NCMECEvent;
51
52export const NCMECIndustryClassification = makeEnumLike([
53 'A1',
54 'A2',
55 'B1',
56 'B2',
57]);
58export type NCMECIndustryClassificationType =
59 keyof typeof NCMECIndustryClassification;
60
61export const NCMECFileAnnotation = makeEnumLike([
62 'ANIME_DRAWING_VIRTUAL_HENTAI',
63 'POTENTIAL_MEME',
64 'VIRAL',
65 'POSSIBLE_SELF_PRODUCTION',
66 'PHYSICAL_HARM',
67 'VIOLENCE_GORE',
68 'BESTIALITY',
69 'LIVE_STREAMING',
70 'INFANT',
71 'GENERATIVE_AI',
72]);
73export type NCMECFileAnnotationType = keyof typeof NCMECFileAnnotation;
74
75export const NCMECIncidentType = makeEnumLike([
76 'Child Pornography (possession, manufacture, and distribution)',
77 'Child Sex Trafficking',
78 'Child Sex Tourism',
79 'Child Sexual Molestation',
80 'Misleading Domain Name',
81 'Misleading Words or Digital Images on the Internet',
82 'Online Enticement of Children for Sexual Acts',
83 'Unsolicited Obscene Material Sent to a Child',
84]);
85export type NCMECIncidentType = keyof typeof NCMECIncidentType;
86
87export const NCMECEmailType = makeEnumLike(['Home', 'Work', 'Business']);
88export type NCMECEmailType = keyof typeof NCMECEmailType;
89
90export type NCMECMediaReport = {
91 id: string;
92 typeId: string;
93 url: string;
94 fileAnnotations: readonly NCMECFileAnnotationType[];
95 industryClassification: NCMECIndustryClassificationType;
96};
97
98type NCMECEventInfo = {
99 eventName: NCMECEventType;
100 dateTime: string;
101};
102
103type IPNCMECEvent = NCMECEventInfo & {
104 ipAddress: string;
105 port?: number;
106 possibleProxy?: boolean;
107};
108
109type DeviceNCMECEvent = NCMECEventInfo & {
110 idType: string;
111 idValue: string;
112};
113
114type NCMECPerson = {
115 phone?: Phone;
116 email?: Email[];
117 firstName?: string;
118 lastName?: string;
119 deviceId?: DeviceNCMECEvent[];
120};
121
122// Key insertion order must match the NCMEC XSD `<fileDetails>` sequence
123// (Appendix C); see the `Report` type comment for why this matters.
124type FileDetails = {
125 fileDetails: {
126 reportId: number;
127 fileId: string;
128 originalFileName?: string;
129 locationOfFile?: string;
130 fileViewedByEsp?: boolean;
131 exifViewedByEsp?: boolean;
132 publiclyAvailable?: boolean;
133 fileRelevance?: 'Reported' | 'Supplemental Reported';
134 fileAnnotations?: FileAnnotations;
135 industryClassification?: NCMECIndustryClassificationType;
136 originalFileHash?: OriginalFileHash[];
137 ipCaptureEvent?: IPNCMECEvent[];
138 deviceId?: DeviceId[];
139 details?: Detail[];
140 additionalInfo?: string[];
141 };
142};
143
144type OriginalFileHash = {
145 _text: string;
146 _attributes: {
147 hashType: string;
148 };
149};
150
151type FileAnnotations = {
152 animeDrawingVirtualHentai?: undefined;
153 potentialMeme?: undefined;
154 viral?: undefined;
155 possibleSelfProduction?: undefined;
156 physicalHarm?: undefined;
157 violenceGore?: undefined;
158 bestiality?: undefined;
159 liveStreaming?: undefined;
160 infant?: undefined;
161 generativeAi?: undefined;
162};
163
164type Detail = {
165 nameValuePair: {
166 name: string;
167 value: string;
168 };
169 type?: 'EXIF' | 'HASH';
170};
171
172type Media = {
173 id: string;
174 typeId: string;
175 url: string;
176 createdAt: string;
177 industryClassification: NCMECIndustryClassificationType;
178 fileAnnotations?: readonly NCMECFileAnnotationType[];
179 /** Pre-built IP event(s); accepts a single event or an array. */
180 ipCaptureEvent?: IPNCMECEvent | readonly IPNCMECEvent[];
181 /** Bare IP from the `ipAddress` field role; appended as a synthesised
182 * `Upload` event. */
183 ipAddress?: string;
184 deviceId?: DeviceNCMECEvent[];
185 /** Hashes computed for this URL (typically by HMA at item submission
186 * time). Keyed by hash algorithm name (e.g. `md5`, `pdq`); the value is
187 * the hex-encoded hash. Forwarded to NCMEC as `originalFileHash`
188 * entries with the algorithm name uppercased into the `hashType`
189 * attribute. */
190 hashes?: Record<string, string>;
191};
192
193type NCMECUserParams = {
194 id: string;
195 typeId: string;
196 profilePicture?: string;
197 displayName?: string;
198 /** Pre-built IP event(s); accepts a single event or an array. */
199 ipCaptureEvent?: IPNCMECEvent | readonly IPNCMECEvent[];
200 /** Bare IP from the `ipAddress` field role; appended as a synthesised
201 * `Unknown` event. */
202 ipAddress?: string;
203 /** Bare email from the `email` field role. Used as the
204 * `personOrUserReportedPerson.email` when no external additional-info
205 * endpoint provided one. */
206 email?: string;
207};
208
209export type NCMECReportParams = {
210 reportedUser: NCMECUserParams;
211 orgId: string;
212 media: Media[];
213 threads: readonly NCMECThreadReport[];
214 reviewerId: string;
215 incidentType: string;
216 /** Optional reason for higher urgency; if present must be non-blank and max 3000 chars. */
217 escalateToHighPriority?: string;
218 /** Optional free-text notes; if present must be non-blank and max 3000 chars. */
219 additionalInfo?: string;
220 /** MRT decision id; when set, failures are recorded to `ncmec_reports_errors`. */
221 jobId?: string;
222};
223
224// Key insertion order in these object literals must match the NCMEC XSD
225// sequence (Appendix B): js2xml emits children in insertion order, and NCMEC
226// rejects out-of-order submissions with responseCode=4100.
227type Report = {
228 report: {
229 incidentSummary: {
230 incidentType: NCMECIncidentType;
231 escalateToHighPriority?: string;
232 incidentDateTime: string;
233 incidentDateTimeDescription?: string;
234 };
235 internetDetails?: (
236 | {
237 webPageIncident: {
238 url?: string;
239 additionalInfo?: string;
240 thirdPartyHostedContent?: boolean;
241 };
242 }
243 | {
244 emailIncident: {
245 emailAddress?: string[];
246 content?: string;
247 additionalInfo?: string;
248 };
249 }
250 | {
251 newsgroupIncident: {
252 name?: string;
253 emailAddress?: string[];
254 content?: string;
255 additionalInfo?: string;
256 };
257 }
258 | {
259 chatImIncident: {
260 chatClient?: string;
261 chatRoomName?: string;
262 content?: string;
263 additionalInfo?: string;
264 };
265 }
266 | {
267 onlineGamingIncident: {
268 gameName?: string;
269 console?: string;
270 content?: string;
271 additionalInfo?: string;
272 };
273 }
274 | {
275 cellPhoneIncident: {
276 phoneNumber?: Phone;
277 latitude?: number;
278 longitude?: number;
279 additionalInfo?: string;
280 };
281 }
282 | {
283 nonInternetIncident: {
284 locationName?: string;
285 incidentAddress?: Address[];
286 additionalInfo?: string;
287 };
288 }
289 | {
290 peer2peerIncident: {
291 client?: string;
292 ipCaptureEvent?: IpCaptureEvent[];
293 additionalInfo?: string;
294 };
295 }
296 )[];
297 lawEnforcement?: {
298 agencyName: string;
299 caseNumber?: string;
300 officerContact: NCMECPerson;
301 reportedToLe?: boolean;
302 servedLegalProcessDomestic?: boolean;
303 servedLegalPorcessInternational?: boolean;
304 };
305 reporter: {
306 reportingPerson: NCMECPerson;
307 contactPerson?: NCMECPerson;
308 companyTemplate?: string;
309 termsOfService?: string;
310 legalURL?: string;
311 };
312 personOrUserReported?: {
313 personOrUserReportedPerson?: NCMECPerson;
314 vehicleDescription?: string;
315 espIdentifier?: string;
316 espService?: string;
317 screenName?: string;
318 displayName?: string[];
319 profileUrl?: string[];
320 ipCaptureEvent?: IpCaptureEvent[];
321 deviceId?: DeviceId[];
322 thirdPartyUserReported?: boolean;
323 priorCTReports?: number[];
324 groupIdentifier?: string;
325 estimatedLocation?: EstimatedLocation;
326 additionalInfo?: string;
327 };
328 intendedRecipient?: {
329 intendedRecipientPerson: NCMECPerson;
330 espIdentifier?: string;
331 espService?: string;
332 screenName?: string;
333 displayName?: string[];
334 profileUrl?: string[];
335 ipCaptureEvent?: IpCaptureEvent[];
336 deviceId?: DeviceId[];
337 priorCTReports?: number[];
338 accountTemporarilyDisabled?: boolean;
339 accountPermanentlyDisabled?: boolean;
340 estimatedLocation?: EstimatedLocation;
341 additionalInfo?: string;
342 }[];
343 victim?: {
344 victimPerson: NCMECPerson;
345 espIdentifier?: string;
346 espService?: string;
347 screenName?: string;
348 displayName?: string[];
349 profileUrl?: string[];
350 ipCaptureEvent?: IpCaptureEvent[];
351 deviceId?: DeviceId[];
352 schoolName?: string;
353 priorCTReports?: number[];
354 estimatedLocation?: EstimatedLocation;
355 additionalInfo?: string;
356 }[];
357 additionalInfo?: string;
358 };
359};
360
361type EstimatedLocation = {
362 city?: string;
363 region?: string;
364 countryCode: string;
365 verified?: boolean;
366 timestamp?: string;
367};
368
369type DeviceId = (
370 | { idType: string; idValue: string }
371 | { idType: undefined; idValue: undefined }
372) & {
373 eventName?: NCMECEventType;
374 dateTime?: string;
375};
376
377type Phone = {
378 // _text should be the phone number
379 _text: string;
380 type?: 'Mobile' | 'Home' | 'Business' | 'Work' | 'Fax' | 'Internet';
381 verified?: boolean;
382 verificationDate?: string;
383 countryCallingCode?: string;
384 extension?: string;
385};
386
387type Email = {
388 // _text should be the email
389 _text: string;
390 _attributes?: {
391 type?: NCMECEmailType;
392 verified?: boolean;
393 verificationDate?: string;
394 };
395};
396
397type Address = {
398 address?: string;
399 city?: string;
400 zipCode?: string;
401 state?: string;
402 nonUsaState?: string;
403 country?: string;
404 type?: string;
405};
406
407type IpCaptureEvent = {
408 ipAddress: string;
409 eventName?: NCMECEventType;
410 dateTime?: string;
411 possibleProxy?: boolean;
412 port?: number;
413};
414
415const NCMEC_INTERNET_DETAIL_TYPES = [
416 'WEB_PAGE',
417 'EMAIL',
418 'NEWSGROUP',
419 'CHAT_IM',
420 'ONLINE_GAMING',
421 'CELL_PHONE',
422 'NON_INTERNET',
423 'PEER_TO_PEER',
424] as const;
425type NcmecInternetDetailTypeSetting =
426 (typeof NCMEC_INTERNET_DETAIL_TYPES)[number];
427
428/** Extract NCMEC's `responseCode`/`responseDescription` from any wrapper
429 * element (`reportResponse`, `uploadResponse`, ...). Either field may be
430 * `undefined` when not present. */
431function extractNcmecResponseStatus(body: unknown): {
432 responseCode?: string;
433 responseDescription?: string;
434} {
435 if (!body || typeof body !== 'object') {
436 return {};
437 }
438 const readText = (value: unknown): string | undefined => {
439 if (typeof value === 'string') {
440 return value;
441 }
442 if (
443 value &&
444 typeof value === 'object' &&
445 '_text' in (value as Record<string, unknown>)
446 ) {
447 const inner = (value as { _text: unknown })._text;
448 return typeof inner === 'string' ? inner : undefined;
449 }
450 return undefined;
451 };
452 const truncate = (value: string | undefined): string | undefined =>
453 value && value.length > 200 ? `${value.slice(0, 200)}…` : value;
454
455 for (const wrapper of Object.values(body as Record<string, unknown>)) {
456 if (!wrapper || typeof wrapper !== 'object') {
457 continue;
458 }
459 const obj = wrapper as Record<string, unknown>;
460 const responseCode = readText(obj.responseCode);
461 const responseDescription = readText(obj.responseDescription);
462 if (responseCode != null || responseDescription != null) {
463 return {
464 responseCode,
465 responseDescription: truncate(responseDescription),
466 };
467 }
468 }
469 return {};
470}
471
472/** Error message for a failed CyberTip response. In production only NCMEC's
473 * documented codes are surfaced (the body may echo reportable content); the
474 * full truncated body is appended only with `NCMEC_DEBUG=1`. */
475export function summarizeCyberTipFailure(
476 route: string,
477 status: number,
478 body: unknown,
479): string {
480 const { responseCode, responseDescription } =
481 extractNcmecResponseStatus(body);
482 const parts: string[] = [
483 `CyberTip request to ${route} failed: status=${status}`,
484 ];
485 if (responseCode != null) {
486 parts.push(`responseCode=${responseCode}`);
487 }
488 if (responseDescription != null) {
489 parts.push(`responseDescription=${responseDescription}`);
490 }
491 if (ncmecDebugEnabled()) {
492 let snippet: string;
493 try {
494 const serialized = jsonStringify(body ?? null);
495 snippet =
496 serialized.length > 500 ? `${serialized.slice(0, 500)}…` : serialized;
497 } catch {
498 snippet = '<unserializable>';
499 }
500 parts.push(`body=${snippet}`);
501 }
502 return parts.join(', ');
503}
504
505/** Clamp a `createdAt` ISO to "now - 1s" if it's ahead of our clock — NCMEC
506 * rejects `<incidentDateTime>` in the future. `value` is always canonicalized
507 * to UTC ISO; `wasClamped` reflects the numeric (not string) comparison so
508 * callers don't misreport plain UTC normalization as a clamp. */
509export function clampIncidentDateTimeToPast(
510 maxCreatedAt: string,
511 nowMs: number = Date.now(),
512): { value: string; wasClamped: boolean } {
513 const maxCreatedAtMs = new Date(maxCreatedAt).getTime();
514 if (Number.isNaN(maxCreatedAtMs)) {
515 throw new Error(
516 `Invalid media createdAt timestamp for incidentDateTime: ${maxCreatedAt}`,
517 );
518 }
519 const ceilingMs = nowMs - 1000;
520 const wasClamped = maxCreatedAtMs > ceilingMs;
521 const finalMs = wasClamped ? ceilingMs : maxCreatedAtMs;
522 return {
523 value: new Date(finalMs).toISOString(),
524 wasClamped,
525 };
526}
527
528/** Build the `ipCaptureEvent` array for an NCMEC person or media block:
529 * webhook events + caller-supplied events + role-IP-synthesised event, in
530 * that order. Returns `undefined` when all sources are empty. */
531export function mergeFieldRoleIpIntoEvents(
532 webhookEvents: readonly IPNCMECEvent[] | undefined,
533 paramEvents: IPNCMECEvent | readonly IPNCMECEvent[] | undefined,
534 roleIpAddress: string | undefined | null,
535 synthesisedEvent: { eventName: NCMECEventType; dateTime: string },
536): IPNCMECEvent[] | undefined {
537 const isEventArray = (
538 v: IPNCMECEvent | readonly IPNCMECEvent[],
539 ): v is readonly IPNCMECEvent[] => Array.isArray(v);
540 const paramEventsArray: readonly IPNCMECEvent[] =
541 paramEvents == null
542 ? []
543 : isEventArray(paramEvents)
544 ? paramEvents
545 : [paramEvents];
546 const trimmedRoleIp =
547 typeof roleIpAddress === 'string' ? roleIpAddress.trim() : '';
548 const events: IPNCMECEvent[] = [
549 ...(webhookEvents ?? []),
550 ...paramEventsArray,
551 ...(trimmedRoleIp !== ''
552 ? [
553 {
554 ipAddress: trimmedRoleIp,
555 eventName: synthesisedEvent.eventName,
556 dateTime: synthesisedEvent.dateTime,
557 },
558 ]
559 : []),
560 ];
561 return events.length > 0 ? events : undefined;
562}
563
564/** Build the NCMEC `originalFileHash[]` shape from both hash sources Coop
565 * has: HMA-computed hashes stored on the item data (keyed by algorithm),
566 * and any single `{ hash, hashType }` returned by the additional-info
567 * webhook for this media. Trims blanks, uppercases the algorithm name into
568 * the `hashType` attribute, drops empty entries, and dedupes on
569 * (`hashType`, hash value) so a webhook that returns the same algorithm as
570 * HMA doesn't produce duplicate entries. Returns undefined when no usable
571 * hashes survive filtering; callers should branch on that to omit the key
572 * entirely rather than serialise an empty array. */
573export function toOriginalFileHashes(opts: {
574 hmaHashes?: Record<string, string>;
575 webhookFileDetails?: { hash: string; hashType: string };
576}): OriginalFileHash[] | undefined {
577 const result: OriginalFileHash[] = [];
578 const seen = new Set<string>();
579 const push = (algorithm: string, hash: string) => {
580 const trimmedHash = typeof hash === 'string' ? hash.trim() : '';
581 const trimmedAlgorithm = algorithm.trim();
582 if (trimmedHash === '' || trimmedAlgorithm === '') return;
583 const hashType = trimmedAlgorithm.toUpperCase();
584 const key = `${hashType} ${trimmedHash}`;
585 if (seen.has(key)) return;
586 seen.add(key);
587 result.push({ _text: trimmedHash, _attributes: { hashType } });
588 };
589 if (opts.hmaHashes) {
590 for (const [algorithm, hash] of Object.entries(opts.hmaHashes)) {
591 push(algorithm, hash);
592 }
593 }
594 if (opts.webhookFileDetails) {
595 push(opts.webhookFileDetails.hashType, opts.webhookFileDetails.hash);
596 }
597 return result.length > 0 ? result : undefined;
598}
599
600/** Resolve the email(s) for `personOrUserReportedPerson`. Prefers the
601 * webhook's enriched response (carries NCMEC `type` / `verified` attributes);
602 * falls back to a bare field-role email otherwise. Returns undefined when
603 * neither source has data. */
604export function resolveReportedPersonEmail(
605 webhookEmails: Email[] | undefined,
606 fieldRoleEmail: string | undefined,
607): Email[] | undefined {
608 if (webhookEmails && webhookEmails.length > 0) return webhookEmails;
609 const trimmed = fieldRoleEmail?.trim();
610 if (trimmed) return [{ _text: trimmed }];
611 return undefined;
612}
613
614/** Maps a list of `NCMECFileAnnotationType` enum values to the
615 * `FileAnnotations` element shape expected by the NCMEC `<fileDetails>` XSD
616 * (each annotation becomes an empty self-closing child element).
617 *
618 * Lives at module scope so the `buildFileDetailsObject` helper and the
619 * dry-run dump script can both build file-details XML without instantiating
620 * `NcmecReportingService`. */
621export function fileAnnotationArrayToNCMECFileAnnotation(
622 fileAnnotations?: readonly NCMECFileAnnotationType[],
623): FileAnnotations | undefined {
624 if (!fileAnnotations || fileAnnotations.length === 0) {
625 return undefined;
626 }
627 // Iterating through a table avoids one branch per annotation, which
628 // previously pushed cyclomatic complexity over the configured ESLint limit
629 // and made the function hard to extend when new annotation types are added.
630 const annotationFieldByType: Record<
631 NCMECFileAnnotationType,
632 keyof FileAnnotations
633 > = {
634 [NCMECFileAnnotation.ANIME_DRAWING_VIRTUAL_HENTAI]:
635 'animeDrawingVirtualHentai',
636 [NCMECFileAnnotation.POTENTIAL_MEME]: 'potentialMeme',
637 [NCMECFileAnnotation.VIRAL]: 'viral',
638 [NCMECFileAnnotation.POSSIBLE_SELF_PRODUCTION]: 'possibleSelfProduction',
639 [NCMECFileAnnotation.PHYSICAL_HARM]: 'physicalHarm',
640 [NCMECFileAnnotation.VIOLENCE_GORE]: 'violenceGore',
641 [NCMECFileAnnotation.BESTIALITY]: 'bestiality',
642 [NCMECFileAnnotation.LIVE_STREAMING]: 'liveStreaming',
643 [NCMECFileAnnotation.INFANT]: 'infant',
644 [NCMECFileAnnotation.GENERATIVE_AI]: 'generativeAi',
645 };
646 const result: FileAnnotations = {};
647 for (const annotation of fileAnnotations) {
648 const field = annotationFieldByType[annotation];
649 result[field] = undefined;
650 }
651 return result;
652}
653
654export type BuildSubmitReportObjectInput = {
655 reportParams: NCMECReportParams;
656 /** Reported user's webhook-derived additional info, from the
657 * `ncmec_additional_info_endpoint` webhook (or defaulted to empty when no
658 * endpoint is configured). */
659 userAdditionalInfo: {
660 email?: Email[];
661 screenName?: string;
662 ipCaptureEvent?: IPNCMECEvent[];
663 };
664 /** Slice of `ncmec_reporting.ncmec_org_settings` needed to build the
665 * report envelope. `companyTemplate`, `legalURL` and `reportingPersonEmail`
666 * are required (the caller validated them). */
667 orgSettings: {
668 companyTemplate: string;
669 legalURL: string;
670 defaultInternetDetailType?: string | null;
671 termsOfService?: string | null;
672 contactPersonEmail?: string | null;
673 contactPersonFirstName?: string | null;
674 contactPersonLastName?: string | null;
675 contactPersonPhone?: string | null;
676 /** From `ncmec_org_settings.contact_email`. */
677 reportingPersonEmail: string;
678 /** Optional URL used to populate `webPageIncident.url` when the org's
679 * `defaultInternetDetailType` is WEB_PAGE. */
680 moreInfoUrl?: string | null;
681 };
682 /** Latest media `createdAt`, already clamped to the past. */
683 clampedIncidentDateTime: string;
684 /** Prior accepted NCMEC report IDs for the reported user; renders as `<priorCTReports>`. */
685 priorCTReports?: readonly number[];
686};
687
688/** Build the `Report` envelope NCMEC's `/submit` endpoint expects. Pure: no
689 * DB or HTTP. The caller is responsible for resolving org settings and
690 * webhook-sourced additional info; this function only assembles them in the
691 * XSD-mandated order. Used by `submitReport` and by audit/dump tooling. */
692
693// further decomposition would obscure the XSD-mandated insertion order (see
694// the `Report` type comment) that NCMEC validates against.
695export function buildSubmitReportObject(
696 input: BuildSubmitReportObjectInput,
697): Report {
698 const {
699 reportParams,
700 userAdditionalInfo,
701 orgSettings,
702 clampedIncidentDateTime,
703 priorCTReports,
704 } = input;
705 const emailStringToNCMECEmail = (email: string) => ({ _text: email });
706
707 const incidentType =
708 NCMECIncidentType[reportParams.incidentType as NCMECIncidentType];
709
710 const escalateToHighPriority =
711 reportParams.escalateToHighPriority != null
712 ? reportParams.escalateToHighPriority.trim()
713 : undefined;
714 if (
715 escalateToHighPriority !== undefined &&
716 (escalateToHighPriority === '' || escalateToHighPriority.length > 3000)
717 ) {
718 throw new Error(
719 'escalateToHighPriority must be non-blank when supplied and at most 3000 characters',
720 );
721 }
722
723 const reportAdditionalInfo =
724 reportParams.additionalInfo != null
725 ? reportParams.additionalInfo.trim()
726 : undefined;
727 if (
728 reportAdditionalInfo !== undefined &&
729 (reportAdditionalInfo === '' || reportAdditionalInfo.length > 3000)
730 ) {
731 throw new Error(
732 'additionalInfo must be non-blank when supplied and at most 3000 characters',
733 );
734 }
735
736 const internetDetails = buildInternetDetailsFromOrgSetting(
737 orgSettings.defaultInternetDetailType,
738 orgSettings.moreInfoUrl,
739 );
740
741 const contactPersonEmail = orgSettings.contactPersonEmail?.trim();
742 const contactPersonFirstName = orgSettings.contactPersonFirstName?.trim();
743 const contactPersonLastName = orgSettings.contactPersonLastName?.trim();
744 const contactPersonPhone = orgSettings.contactPersonPhone?.trim();
745 const contactPerson =
746 contactPersonEmail ||
747 contactPersonFirstName ||
748 contactPersonLastName ||
749 contactPersonPhone
750 ? {
751 ...(contactPersonFirstName
752 ? { firstName: contactPersonFirstName }
753 : {}),
754 ...(contactPersonLastName ? { lastName: contactPersonLastName } : {}),
755 ...(contactPersonPhone
756 ? { phone: { _text: contactPersonPhone } }
757 : {}),
758 ...(contactPersonEmail
759 ? { email: [emailStringToNCMECEmail(contactPersonEmail)] }
760 : {}),
761 }
762 : undefined;
763
764 const termsOfService =
765 orgSettings.termsOfService != null &&
766 orgSettings.termsOfService.trim() !== '' &&
767 orgSettings.termsOfService.length <= 3000
768 ? orgSettings.termsOfService.trim()
769 : undefined;
770
771 const reportedPersonEmail = resolveReportedPersonEmail(
772 userAdditionalInfo.email,
773 reportParams.reportedUser.email,
774 );
775 const personOrUserReportedPerson = reportedPersonEmail
776 ? { email: reportedPersonEmail }
777 : undefined;
778
779 // The role IP is a bare string, not a login/upload signal, so `Unknown` is
780 // the safest event-name claim.
781 const reportedUserIpCaptureEvents = mergeFieldRoleIpIntoEvents(
782 userAdditionalInfo.ipCaptureEvent,
783 reportParams.reportedUser.ipCaptureEvent,
784 reportParams.reportedUser.ipAddress,
785 {
786 eventName: NCMECEvent.Unknown,
787 dateTime: clampedIncidentDateTime,
788 },
789 );
790
791 return {
792 report: {
793 incidentSummary: {
794 incidentType,
795 ...(escalateToHighPriority ? { escalateToHighPriority } : {}),
796 incidentDateTime: clampedIncidentDateTime,
797 },
798 ...(internetDetails ? { internetDetails } : {}),
799 reporter: {
800 reportingPerson: {
801 email: [emailStringToNCMECEmail(orgSettings.reportingPersonEmail)],
802 },
803 ...(contactPerson ? { contactPerson } : {}),
804 companyTemplate: orgSettings.companyTemplate,
805 ...(termsOfService ? { termsOfService } : {}),
806 legalURL: orgSettings.legalURL,
807 },
808 personOrUserReported: {
809 ...(personOrUserReportedPerson ? { personOrUserReportedPerson } : {}),
810 espIdentifier: reportParams.reportedUser.id,
811 espService: orgSettings.companyTemplate,
812 screenName: userAdditionalInfo.screenName,
813 ...(reportParams.reportedUser.displayName
814 ? { displayName: [reportParams.reportedUser.displayName] }
815 : {}),
816 ...(reportedUserIpCaptureEvents &&
817 reportedUserIpCaptureEvents.length > 0
818 ? { ipCaptureEvent: reportedUserIpCaptureEvents }
819 : {}),
820 ...(priorCTReports && priorCTReports.length > 0
821 ? { priorCTReports: [...priorCTReports] }
822 : {}),
823 },
824 ...(reportAdditionalInfo !== undefined
825 ? { additionalInfo: reportAdditionalInfo }
826 : {}),
827 },
828 };
829}
830
831export type BuildFileDetailsObjectInput = {
832 reportId: number;
833 fileId: string;
834 /** Optional `<originalFileName>`. Usually derived via `deriveOriginalFileNameFromUrl`. */
835 originalFileName?: string;
836 /** `<fileRelevance>`. Defaults to `'Reported'`. */
837 fileRelevance?: 'Reported' | 'Supplemental Reported';
838 media: Pick<Media, 'industryClassification'> & {
839 fileAnnotations?: readonly NCMECFileAnnotationType[];
840 };
841 additionalInfo: Pick<
842 MediaAdditionalInfo,
843 'publiclyAvailable' | 'ipCaptureEvent' | 'additionalInfo'
844 >;
845 /** Combined HMA + webhook hashes as built by `toOriginalFileHashes`.
846 * Rendered in the `<originalFileHash>` element(s) between
847 * `industryClassification` and `ipCaptureEvent` per the XSD. */
848 originalFileHash?: readonly OriginalFileHash[];
849};
850
851/** Decoded last path segment of `url`, or `undefined` if unavailable.
852 * Logs each fallback path via `ncmecDebugLog` so operators can triage
853 * unexpected URL shapes when `NCMEC_DEBUG=1` is enabled. */
854export function deriveOriginalFileNameFromUrl(url: string): string | undefined {
855 let pathname: string;
856 try {
857 pathname = new URL(url).pathname;
858 } catch {
859 ncmecDebugLog('deriveOriginalFileName.urlParseFailed', { url });
860 return undefined;
861 }
862 const last = pathname
863 .split('/')
864 .filter((s) => s !== '')
865 .pop();
866 if (!last) {
867 ncmecDebugLog('deriveOriginalFileName.emptyPath', { url });
868 return undefined;
869 }
870 try {
871 const decoded = decodeURIComponent(last);
872 if (decoded.length === 0) {
873 ncmecDebugLog('deriveOriginalFileName.emptySegment', { url });
874 return undefined;
875 }
876 return decoded;
877 } catch {
878 // Malformed percent-encoding: keep the raw segment.
879 ncmecDebugLog('deriveOriginalFileName.decodeFailed', { url, raw: last });
880 return last;
881 }
882}
883
884/** Pure builder for the `FileDetails` envelope NCMEC's `/fileinfo`
885 * endpoint expects. Used by `submitReport`'s `#upload` step and by
886 * dry-run tooling that needs to serialize per-media XML without
887 * performing the upload. No DB or HTTP calls. */
888export function buildFileDetailsObject(
889 input: BuildFileDetailsObjectInput,
890): FileDetails {
891 const {
892 reportId,
893 fileId,
894 media,
895 additionalInfo,
896 originalFileHash,
897 originalFileName,
898 } = input;
899 const fileRelevance = input.fileRelevance ?? 'Reported';
900 const fileAnnotations = fileAnnotationArrayToNCMECFileAnnotation(
901 media.fileAnnotations,
902 );
903 return {
904 fileDetails: {
905 reportId,
906 fileId,
907 ...(originalFileName ? { originalFileName } : {}),
908 fileViewedByEsp: true,
909 exifViewedByEsp: true,
910 ...(additionalInfo.publiclyAvailable !== undefined
911 ? { publiclyAvailable: additionalInfo.publiclyAvailable }
912 : {}),
913 fileRelevance,
914 ...(fileAnnotations ? { fileAnnotations } : {}),
915 industryClassification: media.industryClassification,
916 ...(originalFileHash && originalFileHash.length > 0
917 ? { originalFileHash: [...originalFileHash] }
918 : {}),
919 ...(additionalInfo.ipCaptureEvent &&
920 additionalInfo.ipCaptureEvent.length > 0
921 ? {
922 ipCaptureEvent: additionalInfo.ipCaptureEvent.map((it) => ({
923 ipAddress: it.ipAddress,
924 eventName: it.eventName,
925 dateTime: it.dateTime,
926 ...(it.possibleProxy ? { possibleProxy: it.possibleProxy } : {}),
927 ...(it.port ? { port: it.port } : {}),
928 })),
929 }
930 : {}),
931 ...(additionalInfo.additionalInfo
932 ? { additionalInfo: additionalInfo.additionalInfo }
933 : {}),
934 },
935 };
936}
937
938export function buildInternetDetailsFromOrgSetting(
939 defaultInternetDetailType: string | null | undefined,
940 moreInfoUrl: string | null | undefined,
941): Report['report']['internetDetails'] {
942 if (!defaultInternetDetailType?.trim()) {
943 return undefined;
944 }
945 const type =
946 defaultInternetDetailType.trim() as NcmecInternetDetailTypeSetting;
947 if (!NCMEC_INTERNET_DETAIL_TYPES.includes(type)) {
948 return undefined;
949 }
950 // NCMEC returns 4100 if `<url>` contains whitespace or control chars; since
951 // the element is optional per the XSD, omit it unless we have a clean value.
952 // eslint-disable-next-line no-control-regex
953 const URL_INVALID_CHARS = /[\s\u0000-\u001f\u007f]/;
954 const trimmedUrl = moreInfoUrl?.trim();
955 const webPageUrl =
956 trimmedUrl && !URL_INVALID_CHARS.test(trimmedUrl) ? trimmedUrl : undefined;
957 switch (type) {
958 case 'WEB_PAGE':
959 return [
960 {
961 webPageIncident: webPageUrl ? { url: webPageUrl } : {},
962 },
963 ];
964 case 'EMAIL':
965 return [{ emailIncident: {} }];
966 case 'NEWSGROUP':
967 return [{ newsgroupIncident: {} }];
968 case 'CHAT_IM':
969 return [{ chatImIncident: {} }];
970 case 'ONLINE_GAMING':
971 return [{ onlineGamingIncident: {} }];
972 case 'CELL_PHONE':
973 return [{ cellPhoneIncident: {} }];
974 case 'NON_INTERNET':
975 return [{ nonInternetIncident: {} }];
976 case 'PEER_TO_PEER':
977 return [{ peer2peerIncident: {} }];
978 default:
979 return assertUnreachable(type);
980 }
981}
982
983// Because CyberTip always responds with XML and how xml2js works, all of the
984// objects returned by it are objects with _text keys
985type CyberTipSubmitResponse = {
986 reportResponse: {
987 responseCode: { _text: string };
988 responseDescription: { _text: string };
989 reportId: { _text: string };
990 };
991};
992
993type CyberTipUploadResponse = {
994 reportResponse: {
995 responseCode: { _text: string };
996 responseDescription: { _text: string };
997 reportId: { _text: string };
998 fileId: { _text: string };
999 hash: { _text: string };
1000 };
1001};
1002
1003type CyberTipFileDetailsResponse = {
1004 reportResponse: {
1005 responseCode: { _text: string };
1006 responseDescription: { _text: string };
1007 reportId: { _text: string };
1008 };
1009};
1010
1011type CyberTipFinishResponse = {
1012 reportDoneResponse: {
1013 responseCode: { _text: string };
1014 reportId: { _text: string };
1015 files: {
1016 fileId: { _text: string };
1017 }[];
1018 };
1019};
1020
1021type CyberTipAuth = {
1022 username: string;
1023 password: string;
1024};
1025
1026type EmailResponse = {
1027 email: string;
1028 type?: NCMECEmailType;
1029 verified?: boolean;
1030 verificationDate?: string;
1031};
1032
1033type NcmecAdditionalInfoResponse = {
1034 users: {
1035 id: string;
1036 typeId: string;
1037 email?: EmailResponse[];
1038 screenName?: string;
1039 ipCaptureEvent?: IPNCMECEvent[];
1040 data?: RawItemData;
1041 }[];
1042 media?: {
1043 id: string;
1044 typeId: string;
1045 ipCaptureEvent?: IPNCMECEvent[];
1046 additionalInfo?: string[];
1047 fileName?: string;
1048 missing?: boolean;
1049 publiclyAvailable?: boolean;
1050 fileDetails?: {
1051 hash: string;
1052 hashType: string;
1053 };
1054 }[];
1055 messages?: {
1056 id: string;
1057 typeId: string;
1058 ipAddress: string;
1059 }[];
1060 additionalFiles?: {
1061 fileUrl: string;
1062 additionalInfo?: string[];
1063 fileName?: string;
1064 }[];
1065 additionalInfo?: string;
1066};
1067
1068type NcmecAdditionalInfo = {
1069 users: {
1070 id: string;
1071 typeId: string;
1072 email?: Email[];
1073 screenName?: string;
1074 ipCaptureEvent?: IPNCMECEvent[];
1075 data?: RawItemData;
1076 }[];
1077 media: MediaAdditionalInfo[];
1078 additionalFiles?: FileAdditionalInfo[];
1079 additionalInfo?: string;
1080};
1081
1082type MediaAdditionalInfo = {
1083 id: string;
1084 typeId: string;
1085 ipCaptureEvent?: IPNCMECEvent[];
1086 additionalInfo?: string[];
1087 fileName?: string;
1088 /** When set, sent to NCMEC in file details (whether the content was publicly viewable). */
1089 publiclyAvailable?: boolean;
1090 /** Optional single hash from the additional-info webhook response.
1091 * Merged with HMA-sourced hashes (see `toOriginalFileHashes`); deduped
1092 * on (`hashType` uppercase, trimmed hash value) so a webhook that
1093 * returns the same algorithm as HMA doesn't produce duplicate entries
1094 * in the outgoing `originalFileHash[]` list. */
1095 fileDetails?: {
1096 hash: string;
1097 hashType: string;
1098 };
1099};
1100
1101type FileAdditionalInfo = {
1102 fileUrl: string;
1103 additionalInfo?: string[];
1104 fileName?: string;
1105};
1106
1107const Ajv = _Ajv as unknown as typeof _Ajv.default;
1108const ajv = new Ajv();
1109
1110const validateIpAddressEvent = {
1111 type: 'array',
1112 items: {
1113 type: 'object',
1114 properties: {
1115 ipAddress: { type: 'string' },
1116 eventName: {
1117 type: 'string',
1118 enum: [
1119 'Login',
1120 'Registration',
1121 'Purchase',
1122 'Upload',
1123 'Other',
1124 'Unknown',
1125 ],
1126 },
1127 dateTime: { type: 'string' },
1128 possibleProxy: { type: 'boolean' },
1129 port: { type: 'integer' },
1130 },
1131 required: ['ipAddress'],
1132 },
1133} as const;
1134
1135type NcmecMessageResponse = {
1136 conversations: {
1137 threadId: string;
1138 typeId: string;
1139 messages: (RawItemSubmission & {
1140 ipAddress: {
1141 ip: string;
1142 port: number;
1143 };
1144 })[];
1145 }[];
1146};
1147
1148const validateNcmecMessages = ajv.compile<NcmecMessageResponse>({
1149 type: 'object',
1150 properties: {
1151 conversations: {
1152 type: 'array',
1153 items: {
1154 type: 'object',
1155 properties: {
1156 threadId: { type: 'string' },
1157 typeId: { type: 'string' },
1158 messages: {
1159 type: 'array',
1160 items: {
1161 type: 'object',
1162 oneOf: [
1163 {
1164 ...rawItemSubmissionSchema.oneOf[0],
1165 properties: {
1166 ...rawItemSubmissionSchema.oneOf[0].properties,
1167 ipAddress: {
1168 type: 'object',
1169 properties: {
1170 ip: { type: 'string' },
1171 port: { type: 'integer' },
1172 },
1173 required: ['ip', 'port'],
1174 },
1175 },
1176 required: [
1177 ...rawItemSubmissionSchema.oneOf[0].required,
1178 'ipAddress',
1179 ],
1180 },
1181 {
1182 ...rawItemSubmissionSchema.oneOf[1],
1183 properties: {
1184 ...rawItemSubmissionSchema.oneOf[1].properties,
1185 ipAddress: {
1186 type: 'object',
1187 properties: {
1188 ip: { type: 'string' },
1189 port: { type: 'integer' },
1190 },
1191 required: ['ip', 'port'],
1192 },
1193 },
1194 required: [
1195 ...rawItemSubmissionSchema.oneOf[1].required,
1196 'ipAddress',
1197 ],
1198 },
1199 ],
1200 },
1201 },
1202 },
1203 required: ['threadId', 'typeId', 'messages'],
1204 },
1205 },
1206 },
1207 required: ['conversations'],
1208} as const satisfies JSONSchemaV4<NcmecMessageResponse>);
1209
1210const validateNcmecAdditionalInfo = ajv.compile<NcmecAdditionalInfoResponse>({
1211 type: 'object',
1212 properties: {
1213 users: {
1214 type: 'array',
1215 items: {
1216 type: 'object',
1217 properties: {
1218 id: { type: 'string' },
1219 typeId: { type: 'string' },
1220 screenName: { type: 'string' },
1221 email: {
1222 type: 'array',
1223 items: {
1224 type: 'object',
1225 properties: {
1226 email: { type: 'string' },
1227 verified: { type: 'boolean' },
1228 verificationDate: { type: 'string' },
1229 type: { type: 'string', enum: ['Business', 'Home', 'Work'] },
1230 },
1231 required: ['email'],
1232 },
1233 },
1234 ipCaptureEvent: validateIpAddressEvent,
1235 // NB: the typings break here if we don't have { required: [] },
1236 // but actually putting an empty array for `required` in the runtime
1237 // value breaks request handling, so we just use a cast.
1238 data: { type: 'object' } as unknown as {
1239 type: 'object';
1240 required: [];
1241 },
1242 },
1243 required: ['id', 'typeId'],
1244 },
1245 },
1246 media: {
1247 type: 'array',
1248 items: {
1249 type: 'object',
1250 properties: {
1251 id: { type: 'string' },
1252 typeId: { type: 'string' },
1253 ipCaptureEvent: validateIpAddressEvent,
1254 additionalInfo: {
1255 type: 'array',
1256 items: {
1257 type: 'string',
1258 },
1259 },
1260 fileName: { type: 'string' },
1261 missing: { type: 'boolean' },
1262 publiclyAvailable: { type: 'boolean' },
1263 fileDetails: {
1264 type: 'object',
1265 properties: {
1266 hash: { type: 'string' },
1267 hashType: { type: 'string' },
1268 },
1269 required: ['hash', 'hashType'],
1270 },
1271 },
1272 required: ['id', 'typeId'],
1273 },
1274 },
1275 additionalFiles: {
1276 type: 'array',
1277 items: {
1278 type: 'object',
1279 properties: {
1280 fileUrl: { type: 'string' },
1281 additionalInfo: {
1282 type: 'array',
1283 items: {
1284 type: 'string',
1285 },
1286 },
1287 fileName: { type: 'string' },
1288 },
1289 required: ['fileUrl'],
1290 },
1291 },
1292 messages: {
1293 type: 'array',
1294 items: {
1295 type: 'object',
1296 properties: {
1297 id: { type: 'string' },
1298 typeId: { type: 'string' },
1299 ipAddress: { type: 'string' },
1300 },
1301 required: ['id', 'typeId', 'ipAddress'],
1302 },
1303 },
1304 additionalInfo: {
1305 type: 'string',
1306 },
1307 },
1308 additionalProperties: true,
1309 required: ['users'],
1310} as const satisfies JSONSchemaV4<NcmecAdditionalInfoResponse>);
1311
1312export type NcmecMediaReport = {
1313 id: string;
1314 typeId: string;
1315 xml: string;
1316 ncmecFileId: string;
1317};
1318
1319export type NcmecAdditionalFile = {
1320 xml: string;
1321 ncmecFileId: string;
1322 url: string;
1323};
1324
1325export type NcmecMessagesReport = {
1326 csv: string;
1327 ncmecFileId: string;
1328 fileName: string;
1329};
1330
1331type NcmecReportResult =
1332 | 'ALL_MEDIA_MISSING'
1333 | 'SUCCESS'
1334 | 'UNSUPPORTED_ORG'
1335 | 'FAILURE';
1336
1337const actionsOnReportCreationAndPoliciesSelection = [
1338 'actions_to_run_upon_report_creation as actionsToRunIds',
1339 'policies_applied_to_actions_run_on_report_creation as policyIds',
1340] as const;
1341
1342type ActionsOnReportCreationAndPoliciesSelectionResult =
1343 FixKyselyRowCorrelation<
1344 NcmecReportingServicePg['ncmec_reporting.ncmec_org_settings'],
1345 typeof actionsOnReportCreationAndPoliciesSelection
1346 >;
1347
1348export default class NcmecReporting {
1349 constructor(
1350 private pgQuery: Kysely<NcmecReportingServicePg>,
1351 private pqQueryReadReplica: Kysely<NcmecReportingServicePg>,
1352 private fetchHTTP: Dependencies['fetchHTTP'],
1353 private signingKeyPairService: Dependencies['SigningKeyPairService'],
1354 private moderationConfigService: Dependencies['ModerationConfigService'],
1355 private getItemTypeEventuallyConsistent: Dependencies['getItemTypeEventuallyConsistent'],
1356 private readonly tracer: Dependencies['Tracer'],
1357 ) {}
1358 async hasNCMECReportingEnabled(orgId: string) {
1359 const ncmecOrgSettings = await this.pgQuery
1360 .selectFrom('ncmec_reporting.ncmec_org_settings')
1361 .select(['org_id'])
1362 .where('org_id', '=', orgId)
1363 .executeTakeFirst();
1364 return ncmecOrgSettings?.org_id != null;
1365 }
1366
1367 async getNCMECConfig(
1368 orgId: string,
1369 ): Promise<
1370 NcmecReportingServicePg['ncmec_reporting.ncmec_org_settings'] | undefined
1371 > {
1372 const row = await this.pgQuery
1373 .selectFrom('ncmec_reporting.ncmec_org_settings')
1374 .selectAll()
1375 .where('org_id', '=', orgId)
1376 .executeTakeFirst();
1377 return row as
1378 | NcmecReportingServicePg['ncmec_reporting.ncmec_org_settings']
1379 | undefined;
1380 }
1381
1382 async getNCMECActionsToRunAndPolicies(
1383 orgId: string,
1384 ): Promise<ActionsOnReportCreationAndPoliciesSelectionResult | undefined> {
1385 const row = await this.pgQuery
1386 .selectFrom('ncmec_reporting.ncmec_org_settings')
1387 .select(actionsOnReportCreationAndPoliciesSelection)
1388 .where('org_id', '=', orgId)
1389 .executeTakeFirst();
1390
1391 return row
1392 ? (row satisfies CollapseCases<ActionsOnReportCreationAndPoliciesSelectionResult> as ActionsOnReportCreationAndPoliciesSelectionResult)
1393 : undefined;
1394 }
1395
1396 async getNcmecMessages(
1397 orgId: string,
1398 userId: ItemIdentifier,
1399 reportedMedia: readonly ItemIdentifier[],
1400 ) {
1401 const fetchWithRetries = withRetries(
1402 {
1403 maxRetries: 5,
1404 initialTimeMsBetweenRetries: 5,
1405 maxTimeMsBetweenRetries: 500,
1406 jitter: true,
1407 },
1408 async () => {
1409 const response = await this.fetchHTTP({
1410 url: 'https://tas-infra-ml.net/data/coop/content/pre-preserve/get',
1411 method: 'post',
1412 body: jsonStringify({
1413 userId: userId.id,
1414 typeId: userId.typeId,
1415 reported_messages: reportedMedia,
1416 }),
1417 handleResponseBody: 'as-json',
1418 signWith: this.signingKeyPairService.sign.bind(
1419 this.signingKeyPairService,
1420 orgId,
1421 ),
1422 });
1423 if (!response.ok) {
1424 throw new Error();
1425 }
1426 const responseBody = response.body;
1427 if (!validateNcmecMessages(responseBody)) {
1428 throw new Error(`NCMEC Messages failed validation`);
1429 }
1430 return responseBody;
1431 },
1432 );
1433 const body = await fetchWithRetries();
1434
1435 return Promise.all(
1436 body.conversations.map(async (conversation) => {
1437 const messages = await Promise.all(
1438 conversation.messages.map(async (message) => {
1439 const { error, itemSubmission } =
1440 await rawItemSubmissionToItemSubmission(
1441 await this.moderationConfigService.getItemTypes({
1442 orgId,
1443 directives: { maxAge: 10 },
1444 }),
1445 orgId,
1446 this.getItemTypeEventuallyConsistent,
1447 message,
1448 );
1449 if (error) {
1450 throw error;
1451 }
1452 return {
1453 message: itemSubmission,
1454 ipAddress: message.ipAddress,
1455 };
1456 }),
1457 );
1458 return {
1459 messages: messages.slice(-50), // Get the last 50 items
1460 threadId: conversation.threadId,
1461 threadTypeId: conversation.typeId,
1462 };
1463 }),
1464 );
1465 }
1466
1467 async getNCMECAdditionalInfo(
1468 orgId: string,
1469 reportedUsers: ItemIdentifier[],
1470 reportedMedia: readonly ItemIdentifier[],
1471 ): Promise<NcmecAdditionalInfo | 'ALL_MEDIA_MISSING'> {
1472 const additionalInfoEndpoint =
1473 await this.ncmecAdditionalInfoEndpoint(orgId);
1474
1475 // If no additional info endpoint is configured, return minimal default data
1476 if (!additionalInfoEndpoint) {
1477 return {
1478 users: reportedUsers.map((user) => ({
1479 id: user.id,
1480 typeId: user.typeId,
1481 email: [],
1482 screenName: user.id, // Use ID as fallback
1483 ipCaptureEvent: [],
1484 })),
1485 media: reportedMedia.map((media) => ({
1486 id: media.id,
1487 typeId: media.typeId,
1488 })),
1489 };
1490 }
1491
1492 const response = await this.fetchHTTP({
1493 url: additionalInfoEndpoint,
1494 method: 'post',
1495 body: jsonStringify({
1496 users: reportedUsers,
1497 media: reportedMedia,
1498 }),
1499 handleResponseBody: 'as-json',
1500 signWith: this.signingKeyPairService.sign.bind(
1501 this.signingKeyPairService,
1502 orgId,
1503 ),
1504 });
1505
1506 if (!response.ok) {
1507 throw new Error(
1508 `NCMEC Additional info failed with status: ${response.status}`,
1509 );
1510 }
1511
1512 const responseBody = response.body;
1513 if (!validateNcmecAdditionalInfo(responseBody)) {
1514 throw new Error(`NCMEC Additional info failed validation`);
1515 }
1516
1517 // Validate that we received information from every piece of content we
1518 // requested it for
1519 if (
1520 reportedMedia.some(
1521 (inputMedia) =>
1522 responseBody.media?.find(
1523 (responseMedia) =>
1524 inputMedia.id === responseMedia.id &&
1525 inputMedia.typeId === responseMedia.typeId,
1526 ) === undefined,
1527 ) ||
1528 reportedUsers.some(
1529 (inputUser) =>
1530 responseBody.users.find(
1531 (responseUser) =>
1532 inputUser.id === responseUser.id &&
1533 inputUser.typeId === responseUser.typeId,
1534 ) === undefined,
1535 )
1536 ) {
1537 throw new Error(
1538 `Did not receive additional info back for every user and media`,
1539 );
1540 }
1541
1542 if (
1543 responseBody.media?.filter(
1544 (it) => it.missing === false || it.missing === undefined,
1545 ).length === 0
1546 ) {
1547 return 'ALL_MEDIA_MISSING';
1548 }
1549
1550 // Convert email to the type expected by js2xml
1551 return {
1552 // We shouldn't have to do this omit since it gets overwritten later, but
1553 // the data in users makes this think this could be a JSON
1554 ..._.omit(responseBody, 'users'),
1555 media: responseBody.media?.filter((it) => it.missing !== true) ?? [],
1556 users: responseBody.users.map((user) => ({
1557 ...user,
1558 email: user.email?.map((it) => ({
1559 _text: it.email,
1560 _attributes: {
1561 ..._.omit(it, 'email'),
1562 },
1563 })),
1564 })),
1565 };
1566 }
1567
1568 async getNcmecReports(opts: { orgId: string; reviewerId: string }) {
1569 const { orgId, reviewerId } = opts;
1570 return (
1571 this.pqQueryReadReplica
1572 .selectFrom('ncmec_reporting.ncmec_reports')
1573 .select([
1574 'created_at as ts',
1575 'report_id as reportId',
1576 'user_id as userId',
1577 'user_item_type_id as userItemTypeId',
1578 'reviewer_id as reviewerId',
1579 'reported_media as reportedMedia',
1580 'report_xml as reportXml',
1581 'additional_files as additionalFiles',
1582 'reported_messages as reportedMessages',
1583 'is_test as isTest',
1584 ])
1585 .where('org_id', '=', orgId)
1586 .where((eb) =>
1587 eb.or([
1588 eb('is_test', '=', false),
1589 eb('reviewer_id', '=', reviewerId),
1590 ]),
1591 )
1592 .orderBy('ts', 'desc')
1593 // TODO: Paginate the NCMEC Reports page and make the search function
1594 // issue a new query.
1595 .limit(300)
1596 .execute()
1597 );
1598 }
1599
1600 // Retrieves a list of all users with a valid NCMEC decision, in the trio of
1601 // (user_id, user_item_type_id, org_id) for uniqueness, before an hour before
1602 // it executes, to allow for concurrent decisions to finish executing. Only
1603 // meant to be used in the NCMEC retry script.
1604 async getUsersWithNcmecDecision(opts: { startDate: Date }) {
1605 const { startDate } = opts;
1606 return this.pqQueryReadReplica
1607 .selectFrom('ncmec_reporting.ncmec_reports')
1608 .select([
1609 'user_id as userId',
1610 'user_item_type_id as userItemTypeId',
1611 'org_id as orgId',
1612 ])
1613 .where('created_at', '>=', startDate)
1614 .where((eb) =>
1615 eb.or([eb('is_test', '=', null), eb('is_test', '=', false)]),
1616 )
1617 .groupBy(['user_id', 'user_item_type_id', 'org_id'])
1618 .execute();
1619 }
1620
1621 async getNcmecReportById(opts: { orgId: string; reportId: string }) {
1622 const { orgId, reportId } = opts;
1623 return this.pqQueryReadReplica
1624 .selectFrom('ncmec_reporting.ncmec_reports')
1625 .select([
1626 'created_at as ts',
1627 'report_id as reportId',
1628 'user_id as userId',
1629 'user_item_type_id as userItemTypeId',
1630 'reviewer_id as reviewerId',
1631 'reported_media as reportedMedia',
1632 'report_xml as reportXml',
1633 'additional_files as additionalFiles',
1634 'reported_messages as reportedMessages',
1635 ])
1636 .where('org_id', '=', orgId)
1637 .where('report_id', '=', reportId)
1638 .executeTakeFirst();
1639 }
1640
1641 async #sendUserPreservationRequest(input: {
1642 orgId: string;
1643 user: ItemIdentifier;
1644 reportedMedia: ItemIdentifier[];
1645 reportId: number;
1646 }) {
1647 const { orgId, user, reportedMedia, reportId } = input;
1648 const ncmecPreservationEndpoint =
1649 await this.ncmecPreservationEndpoint(orgId);
1650
1651 if (ncmecPreservationEndpoint == null) {
1652 throw new Error(
1653 'Organization does not have a NCMEC preservation endpoint',
1654 );
1655 }
1656
1657 const fetchWithRetries = withRetries(
1658 {
1659 maxRetries: 5,
1660 initialTimeMsBetweenRetries: 5,
1661 maxTimeMsBetweenRetries: 500,
1662 jitter: true,
1663 },
1664 async () => {
1665 const response = await this.fetchHTTP({
1666 url: ncmecPreservationEndpoint,
1667 method: 'post',
1668 body: jsonStringify({
1669 user,
1670 reportedMedia,
1671 reportId: reportId.toString(),
1672 }),
1673 handleResponseBody: 'discard',
1674 signWith: this.signingKeyPairService.sign.bind(
1675 this.signingKeyPairService,
1676 orgId,
1677 ),
1678 });
1679 if (!response.ok) {
1680 throw new Error();
1681 }
1682 },
1683 );
1684
1685 await fetchWithRetries();
1686 }
1687
1688 async ncmecPreservationEndpoint(orgId: string): Promise<string | undefined> {
1689 const rows = await this.pgQuery
1690 .selectFrom('ncmec_reporting.ncmec_org_settings')
1691 .select(['ncmec_preservation_endpoint'])
1692 .where('org_id', '=', orgId)
1693 .executeTakeFirst();
1694 return rows?.ncmec_preservation_endpoint;
1695 }
1696
1697 async ncmecAdditionalInfoEndpoint(
1698 orgId: string,
1699 ): Promise<string | undefined> {
1700 const rows = await this.pgQuery
1701 .selectFrom('ncmec_reporting.ncmec_org_settings')
1702 .select(['ncmec_additional_info_endpoint'])
1703 .where('org_id', '=', orgId)
1704 .executeTakeFirst();
1705 return rows?.ncmec_additional_info_endpoint;
1706 }
1707
1708 async getUserHasExistingNcmeReport(params: {
1709 orgId: string;
1710 userId: string;
1711 userItemTypeId: string;
1712 }) {
1713 const { orgId, userId, userItemTypeId } = params;
1714 const firstReport = await this.pgQuery
1715 .selectFrom('ncmec_reporting.ncmec_reports')
1716 .select(['report_id'])
1717 .where('org_id', '=', orgId)
1718 .where('user_id', '=', userId)
1719 .where('user_item_type_id', '=', userItemTypeId)
1720 .where('is_test', '=', false)
1721 .executeTakeFirst();
1722 return firstReport != null;
1723 }
1724
1725 /** Prior accepted NCMEC report IDs for the user, most recent first.
1726 * Non-numeric `report_id`s are skipped (XSD requires `xs:integer`). */
1727 async getPriorCTReportIds(params: {
1728 orgId: string;
1729 userId: string;
1730 userItemTypeId: string;
1731 }): Promise<number[]> {
1732 const { orgId, userId, userItemTypeId } = params;
1733 const rows = await this.pgQuery
1734 .selectFrom('ncmec_reporting.ncmec_reports')
1735 .select(['report_id'])
1736 .where('org_id', '=', orgId)
1737 .where('user_id', '=', userId)
1738 .where('user_item_type_id', '=', userItemTypeId)
1739 .where('is_test', '=', false)
1740 .orderBy('created_at', 'desc')
1741 .execute();
1742 return rows
1743 .map((r) => parseInt(r.report_id, 10))
1744 .filter((n) => Number.isFinite(n));
1745 }
1746
1747 async submitReport(
1748 reportParams: NCMECReportParams,
1749 isTest: boolean,
1750 ): Promise<NcmecReportResult> {
1751 return this.tracer.addSpan(
1752 {
1753 resource: 'ncmecReportinService',
1754 operation: 'submitReport',
1755 },
1756 // eslint-disable-next-line complexity
1757 async (span) => {
1758 const logSafeReportParams = {
1759 orgId: reportParams.orgId,
1760 reviewerId: reportParams.reviewerId,
1761 reportedUserId: reportParams.reportedUser.id,
1762 reportedUserTypeId: reportParams.reportedUser.typeId,
1763 mediaCount: reportParams.media.length,
1764 threadsCount: reportParams.threads.length,
1765 incidentType: reportParams.incidentType,
1766 hasEscalation: Boolean(reportParams.escalateToHighPriority?.trim()),
1767 hasAdditionalInfo: Boolean(reportParams.additionalInfo?.trim()),
1768 jobId: reportParams.jobId,
1769 };
1770 span.setAttribute(
1771 `ncmecReportParams`,
1772 jsonStringify(logSafeReportParams),
1773 );
1774 ncmecDebugLog('submitReport.start', {
1775 isTest,
1776 ...logSafeReportParams,
1777 });
1778
1779 // We try/catch this whole process in order to do custom logging on
1780 // failure, since we can't guarantee all traces with exceptions are
1781 // sampled in DD
1782 try {
1783 // These are test accounts that we send to prospective users, and
1784 // they should be able to click "Send to NCMEC" in the UI, but no
1785 // NCMEC report should actually be created.
1786 const testOrgs = ['4def6a77d6a', 'acc701627cb'];
1787
1788 if (!(await this.hasNCMECReportingEnabled(reportParams.orgId))) {
1789 throw new Error(
1790 `NCMEC reports are not enabled for org ${reportParams.orgId}`,
1791 );
1792 }
1793
1794 if (testOrgs.includes(reportParams.orgId)) {
1795 if (reportParams.jobId !== undefined) {
1796 await this.#recordSubmissionError({
1797 jobId: reportParams.jobId,
1798 userId: reportParams.reportedUser.id,
1799 userTypeId: reportParams.reportedUser.typeId,
1800 status: 'PERMANENT_ERROR',
1801 error:
1802 'Org is on the NCMEC test allowlist; reports are suppressed.',
1803 });
1804 }
1805 return 'UNSUPPORTED_ORG';
1806 }
1807
1808 if (reportParams.media.length === 0) {
1809 throw new Error('No media in report');
1810 }
1811 const latestMedia = _.maxBy(reportParams.media, (m) => {
1812 const ms = Date.parse(m.createdAt);
1813 if (Number.isNaN(ms)) {
1814 throw new Error(
1815 `Invalid media createdAt timestamp for incidentDateTime: ${m.createdAt}`,
1816 );
1817 }
1818 return ms;
1819 });
1820 if (latestMedia === undefined) {
1821 throw new Error('No media in report');
1822 }
1823 const maxCreatedAt = latestMedia.createdAt;
1824
1825 const { value: clampedIncidentDateTime, wasClamped } =
1826 clampIncidentDateTimeToPast(maxCreatedAt);
1827 if (wasClamped) {
1828 ncmecDebugLog('incidentDateTime.clamped', {
1829 originalCreatedAt: maxCreatedAt,
1830 clampedTo: clampedIncidentDateTime,
1831 });
1832 }
1833
1834 const cybertipAuthenticationCredentials =
1835 await this.getCybertipAuthenticationCredentials(reportParams.orgId);
1836 if (!cybertipAuthenticationCredentials) {
1837 throw new Error('org id not found');
1838 }
1839
1840 const queryResponse = await this.pgQuery
1841 .selectFrom('ncmec_reporting.ncmec_org_settings')
1842 .select([
1843 'company_template as companyTemplate',
1844 'legal_url as legalURL',
1845 'default_internet_detail_type as defaultInternetDetailType',
1846 'terms_of_service as termsOfService',
1847 'contact_person_email as contactPersonEmail',
1848 'contact_person_first_name as contactPersonFirstName',
1849 'contact_person_last_name as contactPersonLastName',
1850 'contact_person_phone as contactPersonPhone',
1851 ])
1852 .where('org_id', '=', reportParams.orgId)
1853 .executeTakeFirst();
1854
1855 if (
1856 !queryResponse ||
1857 !queryResponse.companyTemplate ||
1858 !queryResponse.legalURL
1859 ) {
1860 throw new Error('Insufficient settings');
1861 }
1862
1863 if (isTest === false) {
1864 const hasExistingReport = await this.getUserHasExistingNcmeReport({
1865 orgId: reportParams.orgId,
1866 userId: reportParams.reportedUser.id,
1867 userItemTypeId: reportParams.reportedUser.typeId,
1868 });
1869 if (hasExistingReport) {
1870 throw new Error(
1871 `User with ID: ${reportParams.reportedUser.id} has existing report`,
1872 );
1873 }
1874 }
1875
1876 const additionalInfo = await this.getNCMECAdditionalInfo(
1877 reportParams.orgId,
1878 [
1879 {
1880 id: reportParams.reportedUser.id,
1881 typeId: reportParams.reportedUser.typeId,
1882 },
1883 ],
1884 reportParams.media
1885 .map((media) => ({
1886 id: media.id,
1887 typeId: media.typeId,
1888 }))
1889 // If the user's profile picture or any other media on the user is
1890 // reported, it will manifest as the report. Filter this out and
1891 // assume that there are no IP events/additional info.
1892 .filter(
1893 (media) =>
1894 !(
1895 media.id === reportParams.reportedUser.id &&
1896 media.typeId === reportParams.reportedUser.typeId
1897 ),
1898 ),
1899 );
1900 if (additionalInfo === 'ALL_MEDIA_MISSING') {
1901 if (reportParams.jobId !== undefined) {
1902 await this.#recordSubmissionError({
1903 jobId: reportParams.jobId,
1904 userId: reportParams.reportedUser.id,
1905 userTypeId: reportParams.reportedUser.typeId,
1906 status: 'PERMANENT_ERROR',
1907 error:
1908 'All reportable media is missing from storage; nothing to submit.',
1909 });
1910 }
1911 return 'ALL_MEDIA_MISSING';
1912 }
1913 // This should be validated in getNCMECAdditionalInfo so the ! is safe
1914 const userAdditionalInfo = additionalInfo.users.find(
1915 (it) =>
1916 it.id === reportParams.reportedUser.id &&
1917 it.typeId === reportParams.reportedUser.typeId,
1918 )!;
1919 const ncmecConfig = await this.getNCMECConfig(reportParams.orgId);
1920
1921 const reportingPersonEmail = ncmecConfig?.contact_email?.trim();
1922 if (!reportingPersonEmail) {
1923 throw new Error(
1924 'NCMEC report requires a non-empty reporter contact email; configure it in Settings → NCMEC.',
1925 );
1926 }
1927
1928 // Skip for test submissions: prod and exttest report IDs don't
1929 // cross-reference.
1930 const priorCTReports = isTest
1931 ? []
1932 : await this.getPriorCTReportIds({
1933 orgId: reportParams.orgId,
1934 userId: reportParams.reportedUser.id,
1935 userItemTypeId: reportParams.reportedUser.typeId,
1936 });
1937
1938 const report = buildSubmitReportObject({
1939 reportParams,
1940 userAdditionalInfo,
1941 orgSettings: {
1942 companyTemplate: queryResponse.companyTemplate,
1943 legalURL: queryResponse.legalURL,
1944 defaultInternetDetailType:
1945 queryResponse.defaultInternetDetailType,
1946 termsOfService: queryResponse.termsOfService,
1947 contactPersonEmail: queryResponse.contactPersonEmail,
1948 contactPersonFirstName: queryResponse.contactPersonFirstName,
1949 contactPersonLastName: queryResponse.contactPersonLastName,
1950 contactPersonPhone: queryResponse.contactPersonPhone,
1951 reportingPersonEmail,
1952 moreInfoUrl: ncmecConfig?.more_info_url,
1953 },
1954 clampedIncidentDateTime,
1955 priorCTReports,
1956 });
1957
1958 // For the five actions here
1959 // 1. #submit
1960 // 2. #upload
1961 // 3. #uploadAdditionalFile
1962 // 4. #finish
1963 // 5. #sendUserPreservationRequest
1964 // we should error and mark the span as failed if any single
1965 // call fails.
1966 // These 3 functions utilize #sendCyberTipRequest, which retries
1967 // each request in the event of an initial error to lower the
1968 // likelihood that network or other transient errors blow the
1969 // whole process up.
1970
1971 const { reportId, xml } = await this.#submit(
1972 report,
1973 cybertipAuthenticationCredentials,
1974 isTest,
1975 );
1976
1977 const reportedMedia = await Promise.all(
1978 reportParams.media.map(async (media) => {
1979 const mediaAdditionalInfo = additionalInfo.media.find(
1980 (it) => it.id === media.id && it.typeId === media.typeId,
1981 ) ?? {
1982 id: media.id,
1983 typeId: media.typeId,
1984 additionalInfo: [],
1985 ipCaptureEvent: [],
1986 };
1987 const mergedMediaInfo = {
1988 ...mediaAdditionalInfo,
1989 ipCaptureEvent: mergeFieldRoleIpIntoEvents(
1990 mediaAdditionalInfo.ipCaptureEvent,
1991 media.ipCaptureEvent,
1992 media.ipAddress,
1993 {
1994 eventName: NCMECEvent.Upload,
1995 dateTime: media.createdAt,
1996 },
1997 ),
1998 };
1999 return this.#upload(
2000 reportId,
2001 media,
2002 cybertipAuthenticationCredentials,
2003 mergedMediaInfo,
2004 isTest,
2005 );
2006 }),
2007 );
2008
2009 const additionalFiles = (
2010 additionalInfo.additionalFiles
2011 ? await Promise.all(
2012 additionalInfo.additionalFiles.map(async (additionalFile) =>
2013 this.#uploadAdditionalFile(
2014 reportId,
2015 cybertipAuthenticationCredentials,
2016 additionalFile,
2017 isTest,
2018 ),
2019 ),
2020 )
2021 : []
2022 ).flat();
2023
2024 const threadCsvs = await this.#uploadThreadCsvs(
2025 reportId,
2026 reportParams.threads,
2027 cybertipAuthenticationCredentials,
2028 isTest,
2029 );
2030
2031 await this.#finish(
2032 reportId,
2033 cybertipAuthenticationCredentials,
2034 isTest,
2035 );
2036
2037 await this.pgQuery
2038 .insertInto('ncmec_reporting.ncmec_reports')
2039 .values({
2040 org_id: reportParams.orgId,
2041 report_id: reportId,
2042 user_id: reportParams.reportedUser.id,
2043
2044 user_item_type_id: reportParams.reportedUser.typeId,
2045 reviewer_id: reportParams.reviewerId,
2046 // Safe to cast as a non empty array because of the createdAt check above
2047 reported_media: reportedMedia as NonEmptyArray<NcmecMediaReport>,
2048 report_xml: xml,
2049 additional_files: additionalFiles,
2050 reported_messages: threadCsvs,
2051 incident_type: reportParams.incidentType,
2052 is_test: isTest,
2053 })
2054 .execute();
2055
2056 if (ncmecConfig?.ncmec_preservation_endpoint && isTest === false) {
2057 await this.#sendUserPreservationRequest({
2058 orgId: reportParams.orgId,
2059 user: {
2060 id: reportParams.reportedUser.id,
2061 typeId: reportParams.reportedUser.typeId,
2062 },
2063 reportedMedia: reportParams.media.map((media) => ({
2064 id: media.id,
2065 typeId: media.typeId,
2066 })),
2067 reportId: parseInt(reportId),
2068 });
2069 }
2070 return 'SUCCESS';
2071 } catch (e) {
2072 // We are intentionally using logErrorJson instead of relying on
2073 // safeTracer's logging because those logs are sampled in DD. For
2074 // NCMEC submission errors we need to record all failures and be
2075 // able to see the logs.
2076 // eslint-disable-next-line no-restricted-syntax
2077 logErrorJson({
2078 error: e,
2079 message: jsonStringify({
2080 reportParams: logSafeReportParams,
2081 isTest,
2082 }),
2083 });
2084 span.recordException(e as Exception);
2085 if (reportParams.jobId !== undefined) {
2086 await this.#recordSubmissionError({
2087 jobId: reportParams.jobId,
2088 userId: reportParams.reportedUser.id,
2089 userTypeId: reportParams.reportedUser.typeId,
2090 error: summarizeNcmecErrorForReviewer(e),
2091 });
2092 }
2093 return 'FAILURE';
2094 }
2095 },
2096 );
2097 }
2098
2099 async #uploadAdditionalFile(
2100 reportId: string,
2101 cybertipAuthenticationCredentials: CyberTipAuth,
2102 additionalFileInfo: {
2103 fileUrl: string;
2104 additionalInfo?: string[] | undefined;
2105 fileName?: string;
2106 },
2107 isTest: boolean,
2108 ): Promise<NcmecAdditionalFile> {
2109 const downloadWithRetries = withRetries(
2110 {
2111 maxRetries: 5,
2112 initialTimeMsBetweenRetries: 5,
2113 maxTimeMsBetweenRetries: 500,
2114 jitter: true,
2115 },
2116 async () => {
2117 // TODO: Handle when this fails because of Unidici's memory limit
2118 const response = await this.fetchHTTP({
2119 url: additionalFileInfo.fileUrl,
2120 method: 'get',
2121 handleResponseBody: 'as-readable-stream',
2122 maxResponseSize: 'unlimited',
2123 iWillConsumeTheResponseBodyStreamQuicklyToAvoidACrash: true,
2124 });
2125
2126 if (!response.ok || !response.body) {
2127 throw new Error(
2128 `Cannot download media from ${additionalFileInfo.fileUrl}`,
2129 );
2130 }
2131
2132 return this.#sendCyberTipRequest({
2133 cybertipAuthenticationCredentials,
2134 body: makeFormDataLikeWithStreams({
2135 id: reportId,
2136 file: {
2137 data: response.body,
2138 fileName: additionalFileInfo.fileName,
2139 },
2140 }),
2141 route: '/upload',
2142 includeContentType: false, // remove ContentType header
2143 isTest,
2144 });
2145 },
2146 );
2147 const response = await downloadWithRetries();
2148
2149 const responseJson = response.body as CyberTipUploadResponse;
2150 if (responseJson.reportResponse.responseCode._text !== '0') {
2151 throw new Error('NCMEC file upload failed.');
2152 }
2153 const fileId = responseJson.reportResponse.fileId._text;
2154 const fileXml = await this.#uploadFileDetails(
2155 {
2156 fileDetails: {
2157 reportId: parseInt(reportId),
2158 fileId,
2159 fileViewedByEsp: true,
2160 fileRelevance: 'Supplemental Reported',
2161 additionalInfo: additionalFileInfo.additionalInfo,
2162 },
2163 },
2164 cybertipAuthenticationCredentials,
2165 isTest,
2166 );
2167 return {
2168 ncmecFileId: fileId,
2169 xml: fileXml,
2170 url: additionalFileInfo.fileUrl,
2171 };
2172 }
2173
2174 async #submit(
2175 report: Report,
2176 cybertipAuthenticationCredentials: CyberTipAuth,
2177 isTest: boolean,
2178 ) {
2179 const reportXML = js2xml(report, { compact: true });
2180
2181 ncmecDebugLog('submit.xml', { isTest, length: reportXML.length });
2182 await ncmecDebugDump(
2183 `${isTest ? 'TEST-' : 'PROD-'}${new Date()
2184 .toISOString()
2185 .replace(/[:.]/g, '-')}-submit.xml`,
2186 reportXML,
2187 );
2188
2189 const response = await this.#sendCyberTipRequest({
2190 cybertipAuthenticationCredentials,
2191 body: reportXML,
2192 route: '/submit',
2193 isTest,
2194 });
2195
2196 const responseJson = response.body as CyberTipSubmitResponse;
2197 if (responseJson.reportResponse.responseCode._text !== '0') {
2198 throw new Error(
2199 `NCMEC report submission failed: responseCode=${responseJson.reportResponse.responseCode._text}`,
2200 );
2201 }
2202
2203 return {
2204 reportId: responseJson.reportResponse.reportId._text,
2205 xml: reportXML,
2206 };
2207 }
2208
2209 async #upload(
2210 reportId: string,
2211 media: Media,
2212 cybertipAuthenticationCredentials: CyberTipAuth,
2213 additionalInfo: MediaAdditionalInfo,
2214 isTest: boolean,
2215 ) {
2216 // TODO: Handle when this fails because of Unidici's memory limit
2217 const downloadWithRetries = withRetries(
2218 {
2219 maxRetries: 5,
2220 initialTimeMsBetweenRetries: 5,
2221 maxTimeMsBetweenRetries: 500,
2222 jitter: true,
2223 },
2224 async () => {
2225 const response = await this.fetchHTTP({
2226 url: media.url,
2227 method: 'get',
2228 handleResponseBody: 'as-readable-stream',
2229 maxResponseSize: 'unlimited',
2230 iWillConsumeTheResponseBodyStreamQuicklyToAvoidACrash: true,
2231 });
2232
2233 if (!response.ok || !response.body) {
2234 throw new Error(`Cannot download media from ${media.url}`);
2235 }
2236
2237 return this.#sendCyberTipRequest({
2238 cybertipAuthenticationCredentials,
2239 body: makeFormDataLikeWithStreams({
2240 id: reportId,
2241 file: {
2242 data: response.body,
2243 fileName: additionalInfo.fileName,
2244 },
2245 }),
2246 route: '/upload',
2247 includeContentType: false,
2248 isTest,
2249 });
2250 },
2251 );
2252
2253 const response = await downloadWithRetries();
2254 const responseJson = response.body as CyberTipUploadResponse;
2255 if (responseJson.reportResponse.responseCode._text !== '0') {
2256 throw new Error('NCMEC file upload failed.');
2257 }
2258 const fileId = responseJson.reportResponse.fileId._text;
2259 const originalFileHash = toOriginalFileHashes({
2260 hmaHashes: media.hashes,
2261 webhookFileDetails: additionalInfo.fileDetails,
2262 });
2263 const originalFileName =
2264 additionalInfo.fileName ?? deriveOriginalFileNameFromUrl(media.url);
2265 const fileDetailsObject = buildFileDetailsObject({
2266 reportId: parseInt(reportId),
2267 fileId,
2268 ...(originalFileName ? { originalFileName } : {}),
2269 media,
2270 additionalInfo,
2271 ...(originalFileHash ? { originalFileHash } : {}),
2272 });
2273 const xml = await this.#uploadFileDetails(
2274 fileDetailsObject,
2275 cybertipAuthenticationCredentials,
2276 isTest,
2277 );
2278 return {
2279 ncmecFileId: fileId,
2280 id: media.id,
2281 typeId: media.typeId,
2282 xml,
2283 };
2284 }
2285
2286 async #uploadFileDetails(
2287 fileDetails: FileDetails,
2288 cybertipAuthenticationCredentials: CyberTipAuth,
2289 isTest: boolean,
2290 ) {
2291 const fileDetailsXML = js2xml(fileDetails, { compact: true });
2292 ncmecDebugLog('fileinfo.xml', {
2293 isTest,
2294 reportId: fileDetails.fileDetails.reportId,
2295 fileId: fileDetails.fileDetails.fileId,
2296 length: fileDetailsXML.length,
2297 });
2298 await ncmecDebugDump(
2299 `${isTest ? 'TEST-' : 'PROD-'}${new Date()
2300 .toISOString()
2301 .replace(/[:.]/g, '-')}-fileinfo-${fileDetails.fileDetails.fileId}.xml`,
2302 fileDetailsXML,
2303 );
2304 const response = await this.#sendCyberTipRequest({
2305 cybertipAuthenticationCredentials,
2306 body: fileDetailsXML,
2307 route: '/fileinfo',
2308 isTest,
2309 });
2310
2311 const responseJson = response.body as CyberTipFileDetailsResponse;
2312 if (responseJson.reportResponse.responseCode._text !== '0') {
2313 throw new Error('NCMEC file upload failed.');
2314 }
2315 return fileDetailsXML;
2316 }
2317
2318 async #uploadThreadCsvs(
2319 reportId: string,
2320 reportedMedia: readonly NCMECThreadReport[],
2321 cybertipAuthenticationCredentials: CyberTipAuth,
2322 isTest: boolean,
2323 ) {
2324 const escapeCSVField = (field: string | undefined | null): string => {
2325 if (field == null) {
2326 return '';
2327 }
2328 const escapedField = field.replace(/"/g, '""');
2329 return `"${escapedField}"`;
2330 };
2331
2332 const transformToCSV = (
2333 reportedContent: readonly NCMECReportedContentInThread[],
2334 threadId: string,
2335 ) => {
2336 const headers = [
2337 'content',
2338 'src',
2339 'target',
2340 'thread',
2341 'type',
2342 'contentId',
2343 'chat_type',
2344 'ip',
2345 ];
2346 const rows = reportedContent.map((content) => [
2347 escapeCSVField(content.content),
2348 escapeCSVField(content.creatorId),
2349 escapeCSVField(content.targetId),
2350 escapeCSVField(threadId),
2351 escapeCSVField(content.type),
2352 // Only send the content ID if it's not text
2353 content.type !== 'text' ? escapeCSVField(content.contentId) : undefined,
2354 escapeCSVField(content.chatType),
2355 escapeCSVField(content.ipAddress.ip),
2356 ]);
2357
2358 // Join headers and rows
2359 return [headers.join(','), ...rows.map((row) => row.join(','))].join(
2360 '\n',
2361 );
2362 };
2363
2364 return Promise.all(
2365 reportedMedia.map(async (thread) => {
2366 const csvContent = transformToCSV(
2367 thread.reportedContent,
2368 thread.threadId,
2369 );
2370 const csvBlob = new Blob([csvContent], { type: 'text/csv' });
2371 const requestBody = new FormData();
2372 requestBody.append('id', reportId);
2373 requestBody.append('file', csvBlob, `${thread.threadId}.csv`);
2374
2375 const response = await this.#sendCyberTipRequest({
2376 cybertipAuthenticationCredentials,
2377 body: requestBody,
2378 route: '/upload',
2379 includeContentType: false,
2380 isTest,
2381 });
2382
2383 if (!response.ok || response.body == null) {
2384 throw new Error('NCMEC thread CSV upload failed.');
2385 }
2386
2387 const responseJson = response.body as CyberTipUploadResponse;
2388 if (responseJson.reportResponse.responseCode._text !== '0') {
2389 throw new Error('NCMEC thread csv failed.');
2390 }
2391 const fileId = responseJson.reportResponse.fileId._text;
2392 await this.#uploadFileDetails(
2393 {
2394 fileDetails: {
2395 reportId: parseInt(reportId),
2396 fileId,
2397 fileViewedByEsp: true,
2398 additionalInfo: [
2399 thread.threadTypeId === 'c01a3f28dfa'
2400 ? 'File contains transcript of a private message conversation involving suspect.'
2401 : 'File contains transcript of a group message conversation involving suspect.',
2402 ],
2403 },
2404 },
2405 cybertipAuthenticationCredentials,
2406 isTest,
2407 );
2408
2409 return {
2410 csv: csvContent,
2411 ncmecFileId: responseJson.reportResponse.fileId._text,
2412 fileName: `${thread.threadId}.csv`,
2413 };
2414 }),
2415 );
2416 }
2417
2418 async #finish(
2419 reportId: string,
2420 cybertipAuthenticationCredentials: CyberTipAuth,
2421 isTest: boolean,
2422 ) {
2423 ncmecDebugLog('finish.request', { reportId, isTest });
2424 const requestBody = new FormData();
2425 requestBody.append('id', reportId);
2426
2427 const response = await this.#sendCyberTipRequest({
2428 cybertipAuthenticationCredentials,
2429 body: requestBody,
2430 route: '/finish',
2431 includeContentType: false,
2432 isTest,
2433 });
2434
2435 if (!response.ok) {
2436 throw new Error('NCMEC report finish failed.');
2437 }
2438
2439 const responseJson = response.body as CyberTipFinishResponse;
2440 return responseJson.reportDoneResponse.reportId;
2441 }
2442
2443 /** Best-effort write of a failed-submission row. Swallows its own DB errors
2444 * so a follow-on failure can't turn a NCMEC FAILURE into an unhandled
2445 * exception. */
2446 async #recordSubmissionError(opts: {
2447 jobId: string;
2448 userId: string;
2449 userTypeId: string;
2450 error: string;
2451 status?: 'RETRYABLE_ERROR' | 'PERMANENT_ERROR';
2452 }) {
2453 try {
2454 const status = opts.status ?? 'RETRYABLE_ERROR';
2455 // Atomic increment in `doUpdateSet` avoids the read-modify-write race
2456 // when two retries land on the same job_id concurrently.
2457 await this.pgQuery
2458 .insertInto('ncmec_reporting.ncmec_reports_errors')
2459 .values({
2460 job_id: opts.jobId,
2461 user_id: opts.userId,
2462 user_type_id: opts.userTypeId,
2463 status,
2464 last_error: opts.error,
2465 retry_count: 1,
2466 })
2467 .onConflict((oc) =>
2468 oc.columns(['job_id']).doUpdateSet({
2469 retry_count: sql`ncmec_reporting.ncmec_reports_errors.retry_count + 1`,
2470 last_error: opts.error,
2471 status,
2472 }),
2473 )
2474 .execute();
2475 } catch (e: unknown) {
2476 // eslint-disable-next-line no-restricted-syntax
2477 logErrorJson({
2478 error: e,
2479 message: jsonStringify({
2480 context: 'recordSubmissionError',
2481 jobId: opts.jobId,
2482 }),
2483 });
2484 }
2485 }
2486
2487 async #sendCyberTipRequest(input: {
2488 cybertipAuthenticationCredentials: CyberTipAuth;
2489 body: string | FormData | FormDataLikeWithStreams;
2490 route: `/${string}`;
2491 isTest: boolean;
2492 includeContentType?: boolean;
2493 }) {
2494 const {
2495 cybertipAuthenticationCredentials,
2496 body,
2497 route,
2498 isTest,
2499 includeContentType = true,
2500 } = input;
2501 const username = cybertipAuthenticationCredentials.username;
2502 const password = cybertipAuthenticationCredentials.password;
2503
2504 // TODO: update this to https://report.cybertip.org/ispws when we want to submit
2505 // real reports
2506
2507 const sendCyberTipRequestWithRetries = withRetries(
2508 {
2509 maxRetries: 5,
2510 initialTimeMsBetweenRetries: 5,
2511 maxTimeMsBetweenRetries: 500,
2512 jitter: true,
2513 },
2514 async () => {
2515 const url = isTest
2516 ? `https://exttest.cybertip.org/ispws${route}`
2517 : `https://report.cybertip.org/ispws${route}`;
2518 ncmecDebugLog('cybertip.request', {
2519 route,
2520 isTest,
2521 bodyKind: typeof body === 'string' ? 'xml' : 'formData',
2522 bodyLength: typeof body === 'string' ? body.length : undefined,
2523 });
2524 const response = await this.fetchHTTP({
2525 url,
2526 method: 'post',
2527 headers: {
2528 ...(includeContentType ? { 'Content-Type': 'text/xml' } : {}),
2529 Authorization:
2530 'Basic ' +
2531 Buffer.from(`${username}:${password}`).toString('base64'),
2532 },
2533 body,
2534 handleResponseBody: 'as-json-from-xml',
2535 });
2536
2537 ncmecDebugLog('cybertip.response', {
2538 route,
2539 status: response.status,
2540 ok: response.ok,
2541 body: response.body ?? null,
2542 });
2543
2544 if (!response.ok) {
2545 throw new Error(
2546 summarizeCyberTipFailure(route, response.status, response.body),
2547 );
2548 }
2549 return response;
2550 },
2551 );
2552 return sendCyberTipRequestWithRetries();
2553 }
2554
2555 async getCybertipAuthenticationCredentials(orgId: string) {
2556 return this.pgQuery
2557 .selectFrom('ncmec_reporting.ncmec_org_settings')
2558 .select(['username', 'password'])
2559 .where('org_id', '=', orgId)
2560 .executeTakeFirst();
2561 }
2562}