Nightly sync of job postings

#!/usr/bin/env node
// Nightly sync of job postings for a list of company domains.
// usage: node sync-jobs.mjs [companies.txt] [out-dir]
import fs from "node:fs";
import path from "node:path";

const BASE = `https://api-v2.forager.ai/api/${process.env.FORAGER_ACCOUNT_ID}/datastorage`;
const HEADERS = { "X-API-KEY": process.env.FORAGER_API_KEY, "Content-Type": "application/json" };
const JOB_SEARCH_CREDITS = 2;   // per request, https://www.forager.ai/docs/api-overview/credit-pricing
const MAX_PAGES_PER_RUN = 25;   // hard cap: 50 credits per night at most

const [companiesFile = "companies.txt", outDir = "."] = process.argv.slice(2);
const statePath = path.join(outDir, "sync-state.json");
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function forager(method, url, json) {
  let wait = 2000;
  for (let attempt = 0; attempt < 4; attempt++) {
    const resp = await fetch(url, { method, headers: HEADERS, body: json && JSON.stringify(json) });
    if (resp.status === 403) throw new Error(`Forager rejected the API key or Account ID: ${await resp.text()}`);
    if (resp.status === 429 && attempt < 3) {
      const retryAfter = Number(resp.headers.get("retry-after"));
      await sleep(retryAfter > 0 ? retryAfter * 1000 : wait);
      wait *= 2;
      continue;
    }
    if (!resp.ok) throw new Error(`Forager ${method} ${url} -> HTTP ${resp.status}: ${await resp.text()}`);
    return resp.json();
  }
}

function loadState() {
  try {
    return JSON.parse(fs.readFileSync(statePath, "utf8"));
  } catch {
    return { orgIds: {}, seenPostIds: [], lastRunDate: null };
  }
}

const saveState = (state) => fs.writeFileSync(statePath, JSON.stringify(state, null, 2));

async function resolveOrganizations(domains, state) {
  const unresolved = [];
  for (const domain of domains) {
    if (state.orgIds[domain]) continue;
    const data = await forager("GET", `${BASE}/autocomplete/organizations/?q=${encodeURIComponent(domain)}`);
    const hit = data.results.find((r) => r.text.toLowerCase().endsWith(` - ${domain.toLowerCase()}`)) ?? data.results[0];
    if (hit) state.orgIds[domain] = Number(hit.id);
    else unresolved.push(domain);
  }
  return unresolved;
}

async function fetchNewPosts(orgIds, since, seen) {
  const fresh = [];
  let total = Infinity;
  let fetched = 0;
  let pages = 0;
  for (let page = 1; page <= MAX_PAGES_PER_RUN && fetched < total; page++) {
    pages++;
    const data = await forager("POST", `${BASE}/job_search/`, {
      organization_ids: orgIds,
      is_active: true,
      date_featured_start: since,
      page,
    });
    total = data.total_search_results;
    const posts = data.search_results;
    if (!posts.length) break;
    fetched += posts.length;
    for (const post of posts) {
      if (seen.has(post.id)) continue;
      seen.add(post.id);
      fresh.push({
        id: post.id,
        title: post.title,
        url: post.url,
        source: post.source,
        date_featured: post.date_featured,
        is_remote: post.is_remote,
        location: post.location?.name ?? null,
        organization: post.organization?.name ?? null,
        organization_domain: post.organization?.domain ?? null,
      });
    }
  }
  return { fresh, total, fetched, pages };
}

async function main() {
  const domains = fs.readFileSync(companiesFile, "utf8").split("\n").map((l) => l.trim()).filter(Boolean);
  const state = loadState();
  const unresolved = await resolveOrganizations(domains, state);
  saveState(state);
  if (unresolved.length) console.error(`no organization found for: ${unresolved.join(", ")}`);

  const today = new Date().toISOString().slice(0, 10);
  const since = state.lastRunDate ?? today;
  const seen = new Set(state.seenPostIds);
  const orgIds = domains.map((d) => state.orgIds[d]).filter(Boolean);
  const { fresh, total, fetched, pages } = await fetchNewPosts(orgIds, since, seen);

  if (fresh.length) {
    const outFile = path.join(outDir, `new-jobs-${today}.ndjson`);
    fs.appendFileSync(outFile, fresh.map((p) => JSON.stringify(p)).join("\n") + "\n");
    console.log(`wrote ${fresh.length} new posts to ${outFile}`);
  }
  console.log(`${orgIds.length} companies, ${total} active posts since ${since}, ${fresh.length} new, ${pages * JOB_SEARCH_CREDITS} credits spent`);
  if (fetched < total) console.warn(`stopped at the ${MAX_PAGES_PER_RUN}-page cap; ${total - fetched} posts not fetched`);

  state.lastRunDate = today;
  state.seenPostIds = [...seen];
  saveState(state);
}

main().catch((e) => { console.error(e.message); process.exit(1); });

15 2 * * *  cd /srv/forager-jobs && set -a && . ./forager.env && set +a && node sync-jobs.mjs companies.txt ./jobs >> sync.log 2>&1

wrote 4 new posts to jobs/new-jobs-2026-11-01.ndjson
3 companies, 11 active posts since 2026-10-31, 4 new, 4 credits spent

{"id":9183021,"title":"Senior Data Engineer","url":"https://example.com/careers/senior-data-engineer","source":"linkedin","date_featured":"2026-10-31","is_remote":true,"location":"Austin, Texas, United States","organization":"Acme Example","organization_domain":"acme.com"}