// ─── Strata Analysis Engine ─────────────────────────────────────────
// Cross-references an island (cluster of linked URLs) against:
//   1. Carry semantic triples from `carry query --format json`
//   2. Structured carry records from D1 (entities, citations, reasonings)
// Follows carry semantics: connections are source → relation → target.
// If one side is in the island, the other side becomes a new connection.
// Returns structured analysis + new connection records to be created on PDS.

import { readFileSync, existsSync } from "fs";
import { join } from "path";

const CARRY_PATH = process.env.CARRY_PATH || "/data/carry";

// ─── Types ──────────────────────────────────────────────────────────

interface VertexInfo {
  uri: string;
  title: string;
  description: string;
  type: string;
}

interface EdgeInfo {
  uri: string;
  did: string;
  source: string;
  target: string;
  connectionType: string;
  note: string;
}

interface CarryEntity {
  atUri: string;
  name: string;
  entityType: string;
  description?: string;
  url?: string; // The entity's canonical https:// URL
}

interface CarryCitation {
  atUri: string;
  url: string;
  title: string;
  takeaway: string;
  confidence?: string;
}

interface CarryReasoning {
  atUri: string;
  claim: string;
  evidence: string;
  reasoningType?: string;
  url?: string; // The reasoning's source https:// URL
}

interface CarryData {
  entities: CarryEntity[];
  citations: CarryCitation[];
  reasonings: CarryReasoning[];
  connections: CarryConnection[];
}

// Structured carry connection — a semantic triple
interface CarryConnection {
  sourceName: string;   // e.g. "Khayame"
  targetName: string;   // e.g. "Vickers Appreciative Systems"
  sourceUrl: string;    // e.g. "https://link.springer.com/..."
  targetUrl: string;    // e.g. "https://pubadmin.institute/..."
  relation: string;     // e.g. "intellectual_influence"
  context: string;      // e.g. "Khayame draws on Vickers' appreciative systems..."
}

interface NewConnection {
  source: string;       // island vertex URI
  target: string;       // carry record AT URI
  connectionType: string; // e.g. "network.cosmik.connection#annotates"
  note: string;         // why this connection exists
}

interface AnalysisInput {
  did: string;
  island: {
    vertices: VertexInfo[];
    edges: EdgeInfo[];
  };
  carryData?: CarryData;
}

interface AnalysisResult {
  themes: string[];
  highlights: string[];
  tensions: string[];
  openQuestions: string[];
  synthesis: string;
  newConnections: NewConnection[];
}

// ─── Main analysis function ────────────────────────────────────────

export async function analyze(input: AnalysisInput): Promise<AnalysisResult> {
  const { island, carryData } = input;

  // 1. Read carry JSON exports (from `carry query --format json` during sync)
  const carryEntities = readCarryEntities();
  const carryCitations = readCarryCitations();
  const carryConnections = readCarryConnections();
  const entityUrlIndex = buildEntityUrlIndex();

  // 2. Build island URL set for fast lookup
  const islandText = buildIslandText(island);
  const islandUrls = new Set(island.vertices.map(v => v.uri));

  // 3. Extract themes
  const themes = extractThemes(islandText);

  // 4. Match carry data against island — follow carry semantics
  const newConnections = matchCarryData(
    island, islandUrls,
    carryEntities, carryCitations, carryConnections, entityUrlIndex,
    carryData || { entities: [], citations: [], reasonings: [], connections: [] },
  );

  // 6. Flag hot edges
  const highlights = findHotEdges(islandText, island, newConnections);

  // 7. Find tensions from carry reasonings
  const tensions = findTensionsFromReasonings(islandText, carryData?.reasonings || []);

  // Deduplicate newConnections
  const seen = new Set<string>();
  const deduped = newConnections.filter(c => {
    const key = `${c.source}|${c.target}`;
    if (seen.has(key)) return false;
    seen.add(key);
    return true;
  }).slice(0, 20);

  // Regenerate synthesis and openQuestions with deduped count
  const finalQuestions = generateOpenQuestions(islandText, island, highlights, tensions, deduped);
  const finalSynthesis = synthesize(island, themes, highlights, tensions, finalQuestions, deduped);

  return { themes, highlights, tensions, openQuestions: finalQuestions, synthesis: finalSynthesis, newConnections: deduped };
}

// ─── Carry JSON readers ────────────────────────────────────────────
// Reads carry data exported by `carry query --format json` during sync.

interface CarryEntityRecord {
  id: string;
  name: string;
  url?: string;
  type?: string;
  description?: string;
}

interface CarryCitationRecord {
  id: string;
  title: string;
  url?: string;
  takeaway?: string;
}

interface CarryConnectionRecord {
  id: string;
  relation: string;
  source?: string;
  target?: string;
  context?: string;
}

function readCarryJson<T>(filename: string): T[] {
  const path = join(CARRY_PATH, filename);
  try {
    if (!existsSync(path)) return [];
    const data = readFileSync(path, "utf-8");
    return JSON.parse(data) as T[];
  } catch (e: any) {
    console.error(`[carry] Failed to read ${filename}:`, e.message);
    return [];
  }
}

function readCarryEntities(): CarryEntityRecord[] {
  return readCarryJson<CarryEntityRecord>("org.latha.entity.json");
}

function readCarryCitations(): CarryCitationRecord[] {
  return readCarryJson<CarryCitationRecord>("org.latha.citation.json");
}

function readCarryConnections(): CarryConnectionRecord[] {
  return readCarryJson<CarryConnectionRecord>("org.latha.connection.json");
}

// Build entity name → URL lookup from carry export
function buildEntityUrlIndex(): Map<string, string> {
  const index = new Map<string, string>();
  for (const e of readCarryEntities()) {
    const url = e.url?.trim().replace(/^"|"$/g, "");
    if (url && url.startsWith("http")) {
      const name = e.name?.trim().replace(/^"|"$/g, "");
      if (name) index.set(name, url);
    }
  }
  return index;
}

// ─── Build island text corpus ──────────────────────────────────────

function buildIslandText(island: { vertices: VertexInfo[]; edges: EdgeInfo[] }): string {
  const parts: string[] = [];

  for (const v of island.vertices) {
    parts.push(v.title);
    if (v.description) parts.push(v.description);
  }

  for (const e of island.edges) {
    if (e.note) parts.push(e.note);
  }

  return parts.join(" ");
}

// ─── Core mapper: match carry semantic triples against island ──────

function matchCarryData(
  island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
  islandUrls: Set<string>,
  carryEntities: CarryEntityRecord[],
  _carryCitations: CarryCitationRecord[],
  carryConnections: CarryConnectionRecord[],
  entityUrlIndex: Map<string, string>,
  _d1CarryData: CarryData,
): NewConnection[] {
  const connections: NewConnection[] = [];

  // ── 1. Carry connections (semantic triples) ──────────────────────
  // A carry connection is source → relation → target.
  // If one side's URL is in the island, the other side becomes a new connection.
  // If both sides are in the island, it's an existing edge (skip).
  // If neither side is in the island, it's a nearby carry connection
  // that could expand the island if linked.
  for (const conn of carryConnections) {
    const srcName = conn.source || "";
    const tgtName = conn.target || "";
    const srcUrl = entityUrlIndex.get(srcName) || "";
    const tgtUrl = entityUrlIndex.get(tgtName) || "";

    if (!srcUrl && !tgtUrl) continue; // Neither side has a URL — skip

    const srcInIsland = srcUrl && islandUrls.has(srcUrl);
    const tgtInIsland = tgtUrl && islandUrls.has(tgtUrl);

    // Both in island → already connected, skip
    if (srcInIsland && tgtInIsland) continue;

    // One side in island → bridge connection
    if (srcInIsland && tgtUrl) {
      connections.push({
        source: srcUrl,
        target: tgtUrl,
        connectionType: `network.cosmik.connection#${conn.relation}`,
        note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
      });
    } else if (tgtInIsland && srcUrl) {
      connections.push({
        source: tgtUrl,
        target: srcUrl,
        connectionType: `network.cosmik.connection#${conn.relation}`,
        note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
      });
    } else if (srcUrl && tgtUrl) {
      // Neither side in island — emit as a nearby carry connection
      // (both sides have URLs, so it's a real edge that could expand the island)
      connections.push({
        source: srcUrl,
        target: tgtUrl,
        connectionType: `network.cosmik.connection#${conn.relation}`,
        note: conn.context || `${srcName} → ${conn.relation} → ${tgtName}`,
      });
    }
    // One side has URL but the other doesn't — can't create a full connection
  }

  // ── 2. Carry entities with URLs not in island ───────────────────
  // These are entities that the user tracks but aren't yet in the island.
  // We don't try to match them by keyword — we just list them as
  // "nearby entities" for the synthesis to reference.
  // (No NewConnection created — entities without a connection triple
  //  don't have a semantic relation to the island.)

  return connections;
}

// ─── Analysis functions ─────────────────────────────────────────────

function extractThemes(text: string): string[] {
  const keywords: Record<string, string[]> = {
    "open science": ["open science", "open access", "oa", "preprint"],
    "data sharing": ["data sharing", "shared data", "open data", "dataset"],
    "reproducibility": ["reproducib", "replicab", "replication"],
    "decentralization": ["decentraliz", "federat", "self-host"],
    "knowledge graphs": ["knowledge graph", "ontolog", "linked data", "semantic"],
    "ai/ml": ["machine learning", "deep learning", "neural", "llm", "gpt", "claude"],
    "community": ["community", "collective", "cooperative", "commons"],
    "governance": ["governance", "policy", "regulation", "standards"],
    "infrastructure": ["infrastructure", "platform", "tooling", "pipeline"],
    "stigmergy": ["stigmerg", "pheromone", "emergent", "self-organiz"],
    "provenance": ["provenance", "lineage", "traceability", "attribution"],
    "doi": ["doi", "zenodo", "citeable", "citation"],
    "privacy": ["privacy", "encryption", "e2ee", "consent"],
    "sovereignty": ["sovereignty", "self-determin", "autonomy", "agency"],
    "content authenticity": ["content authenticity", "c2pa", "provenance watermark", "deepfake"],
    "at protocol": ["atproto", "at protocol", "bluesky", "pds", "appview"],
    "semantic web": ["semantic web", "rdf", "sparql", "linked data"],
  };

  const lower = text.toLowerCase();
  return Object.entries(keywords)
    .filter(([_, terms]) => terms.some(t => lower.includes(t)))
    .map(([theme]) => theme);
}

function findHotEdges(
  islandText: string,
  island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
  newConnections: NewConnection[],
): string[] {
  const highlights: string[] = [];

  // Score each edge by semantic significance
  const edgeScores = new Map<string, number>();

  for (const edge of island.edges) {
    let score = 0;

    // Edges with notes are more interesting
    if (edge.note && edge.note.length > 10) score += 1;

    // Edges with semantic connection types are more interesting
    const semanticTypes = ["supports", "contradicts", "extends", "contextualizes", "exemplifies"];
    if (semanticTypes.some(t => edge.connectionType.toLowerCase().includes(t))) score += 2;

    // Boost edges whose source or target is also connected to carry data
    for (const nc of newConnections) {
      if (edge.source === nc.source || edge.target === nc.source) score += 3;
      if (edge.source === nc.target || edge.target === nc.target) score += 3;
    }

    if (score > 0) {
      edgeScores.set(edge.uri, score);
    }
  }

  // Return top edges by score
  const sorted = [...edgeScores.entries()].sort((a, b) => b[1] - a[1]);
  for (const [uri] of sorted.slice(0, 10)) {
    highlights.push(uri);
  }

  return highlights;
}

function findTensionsFromReasonings(
  islandText: string,
  reasonings: CarryReasoning[],
): string[] {
  const tensions: string[] = [];
  const lowerSet = tokenize(islandText);

  for (const r of reasonings) {
    const rLower = `${r.claim} ${r.evidence}`.toLowerCase();
    if (rLower.includes("contradict") || rLower.includes("however") || rLower.includes("but") || rLower.includes("tension")) {
      const overlap = keywordOverlap(lowerSet, rLower);
      if (overlap.length >= 1) {
        tensions.push(r.claim.slice(0, 500));
      }
    }
  }

  return tensions.slice(0, 5);
}

function generateOpenQuestions(
  islandText: string,
  island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
  highlights: string[],
  tensions: string[],
  newConnections: NewConnection[],
): string[] {
  const questions: string[] = [];
  const lower = islandText.toLowerCase();

  if (newConnections.length > 0) {
    questions.push(
      `${newConnections.length} carry touchpoints found. How does your existing knowledge reshape this island?`
    );
  }

  if (tensions.length > 0) {
    questions.push(
      `What evidence would resolve the ${tensions.length} tension(s) identified?`
    );
  }

  const vertexCount = island.vertices.length;
  const edgeTypes = new Set(island.edges.map(e => e.connectionType));

  if (edgeTypes.size > 1) {
    questions.push(
      `This island has ${edgeTypes.size} connection types. What's the dominant relationship?`
    );
  }

  if (vertexCount > 5) {
    questions.push(
      `This is a ${vertexCount}-vertex cluster. Are there sub-clusters within it?`
    );
  }

  if (lower.includes("data") || lower.includes("dataset")) {
    questions.push("What data artifacts does this cluster produce, and are they in Zenodo?");
  }
  if (lower.includes("standard") || lower.includes("protocol")) {
    questions.push("Does this cluster propose a new standard, and who are the adopters?");
  }

  return questions.slice(0, 5);
}

function synthesize(
  island: { vertices: VertexInfo[]; edges: EdgeInfo[] },
  themes: string[],
  highlights: string[],
  tensions: string[],
  openQuestions: string[],
  newConnections: NewConnection[],
): string {
  const vertexCount = island.vertices.length;
  const edgeCount = island.edges.length;
  const edgeTypes = new Set(island.edges.map(e => e.connectionType));

  const clusterDesc = `${vertexCount} vertices, ${edgeCount} edges, ${edgeTypes.size} connection types`;

  const themeStr = themes.length > 0
    ? `Themes: ${themes.join(", ")}.`
    : "";

  const touchpointStr = newConnections.length > 0
    ? `${newConnections.length} carry touchpoints — new edges connecting this island to your knowledge base.`
    : "No carry touchpoints found.";

  const highlightStr = highlights.length > 0
    ? `${highlights.length} existing edges flagged as semantically significant.`
    : "";

  const tensionStr = tensions.length > 0
    ? `There ${tensions.length === 1 ? "is 1 tension" : `are ${tensions.length} tensions`} with your existing claims.`
    : "";

  const questionStr = openQuestions.length > 0
    ? `Open questions: ${openQuestions.join(" ")}`
    : "";

  return [`Island (${clusterDesc}).`, themeStr, touchpointStr, highlightStr, tensionStr, questionStr].filter(Boolean).join(" ");
}

// ─── Utilities ──────────────────────────────────────────────────────

const STOP_WORDS = new Set([
  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
  "have", "has", "had", "do", "does", "did", "will", "would", "could",
  "should", "may", "might", "shall", "can", "to", "of", "in", "for",
  "on", "with", "at", "by", "from", "as", "into", "through", "during",
  "before", "after", "above", "below", "between", "and", "but", "or",
  "not", "no", "nor", "so", "if", "then", "that", "this", "it", "its",
  "http", "https", "www", "com", "org",
  // Common vault noise terms
  "checkin", "check", "daily", "today", "yesterday", "tomorrow",
  "morning", "evening", "week", "month", "year",
  "also", "just", "like", "really", "very", "much", "some",
  "more", "most", "other", "about", "which", "when", "what",
  "there", "their", "these", "those", "each", "every", "all",
  "make", "made", "thing", "things", "something", "anything",
  "need", "want", "know", "think", "going", "come", "get",
  "good", "great", "nice", "well", "still", "even", "back",
  "first", "last", "new", "old", "own", "same", "another",
  "many", "such", "only", "over", "than", "too", "very",
  "because", "since", "while", "where", "how", "why", "who",
  "work", "working", "time", "people", "using", "used", "use",
]);

function tokenize(text: string): Set<string> {
  return new Set(
    text.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w))
  );
}


function keywordOverlap(setA: Set<string>, textB: string): string[] {
  const wordsB = textB.toLowerCase().split(/\W+/).filter(w => w.length > 3 && !STOP_WORDS.has(w));
  return wordsB.filter(w => setA.has(w));
}