From c3926c7c558f8155bf4469f1eb78ba59c8814460 Mon Sep 17 00:00:00 2001
From: Caleb Denio <cjdenio44@gmail.com>
Date: Tue, 8 Oct 2024 15:57:57 -0400
Subject: [PATCH] analytics + improve GTFS importer

---
 deno.lock                      |  8 ++-
 gtfs.ts => gtfs.js             | 90 +++++++++++++++++++++-------------
 src/components/MbtaLine.svelte |  2 +-
 src/pages/index.astro          | 11 +++++
 4 files changed, 74 insertions(+), 37 deletions(-)
 rename gtfs.ts => gtfs.js (63%)

diff --git a/deno.lock b/deno.lock
index 02360a7..7007461 100644
--- a/deno.lock
+++ b/deno.lock
@@ -21,7 +21,13 @@
   "workspace": {
     "packageJson": {
       "dependencies": [
-        "npm:astro@^4.15.11"
+        "npm:@astrojs/svelte@^5.7.1",
+        "npm:@astrojs/tailwind@^5.1.1",
+        "npm:astro@^4.15.11",
+        "npm:gsap@^3.12.5",
+        "npm:svelte@^4.2.19",
+        "npm:tailwindcss@^3.4.13",
+        "npm:typescript@^5.6.2"
       ]
     }
   }
diff --git a/gtfs.ts b/gtfs.js
similarity index 63%
rename from gtfs.ts
rename to gtfs.js
index e5a0cf9..736e64e 100644
--- a/gtfs.ts
+++ b/gtfs.js
@@ -1,6 +1,7 @@
 import { TextWriter } from "jsr:@zip-js/zip-js";
 import { ZipReader } from "jsr:@zip-js/zip-js";
 import { parse } from "npm:csv-parse/sync";
+import { parse as parseStream } from "npm:csv-parse";
 
 function indexRecords(records, key) {
   const result = {};
@@ -12,46 +13,66 @@ function indexRecords(records, key) {
   return result;
 }
 
+async function streamRecords(entry, filter) {
+  const records = [];
+
+  const parser = parseStream({ columns: true });
+
+  parser.on("readable", () => {
+    let record;
+    while ((record = parser.read()) !== null) {
+      // console.log(`got record ${record.trip_id}`);
+
+      if (filter(record)) records.push(record);
+    }
+  });
+
+  await entry.getData(
+    new WritableStream({
+      write(chunk) {
+        parser.write(chunk);
+      },
+      close() {
+        parser.end();
+      },
+    })
+  );
+
+  return records;
+}
+
 const reader = new ZipReader((await Deno.open(Deno.args[0])).readable);
 
 const files = await reader.getEntries();
 
-const wantedFiles = [
-  "routes",
-  "trips",
-  "stops",
-  "stop_times",
-  "calendar",
-] as const;
+const wantedFiles = ["routes", "trips", "stops", "calendar"];
 
-const entries: Partial<Record<(typeof wantedFiles)[number], any[]>> = {};
+const entries = {};
 
 console.log("reading files");
 
 await Promise.all(
   files
     .filter((f) => {
-      //@ts-ignore
       return wantedFiles.includes(f.filename.replace(".txt", ""));
     })
     .map(async (f) => {
       const writer = new TextWriter();
-      entries[f.filename.replace(".txt", "") as (typeof wantedFiles)[number]] =
-        parse(await f.getData!(writer), { columns: true });
+      entries[f.filename.replace(".txt", "")] = parse(await f.getData(writer), {
+        columns: true,
+      });
     })
 );
 
-reader.close();
-
 console.log(`Indexing ${entries.stops?.length} stops...`);
 
-function actualStop(stop_id: string): string {
+function actualStop(stop_id) {
   return indexedStops[stop_id]?.parent_station || stop_id;
 }
 
 const indexedStops = indexRecords(entries.stops, "stop_id");
 
-const metroRoutes = entries.routes!.filter(
+const metroRoutes = entries.routes.filter(
   (r) => r.route_type == 1 || r.route_type == 0
 );
 
@@ -59,26 +80,31 @@ const metroRouteIds = metroRoutes.map((r) => r.route_id);
 
 console.log(`Indexing trips...`);
 
-const trips = entries.trips!.filter(
+const trips = entries.trips.filter(
   (t) => t.service_id == "canonical" && metroRouteIds.includes(t.route_id)
 );
 const indexedTrips = indexRecords(trips, "trip_id");
 
-console.log("sorting stop times");
+console.log("fetching stop times");
 
-const stop_times = entries
-  .stop_times!.filter((s) => s.trip_id in indexedTrips)
-  .sort((a, b) => {
-    if (a.trip_id < b.trip_id) {
-      return -1;
-    } else if (a.trip_id > b.trip_id) {
-      return 1;
-    } else {
-      return a.stop_sequence - b.stop_sequence;
-    }
-  });
+entries.stop_times = await streamRecords(
+  files.find((f) => f.filename == "stop_times.txt"),
+  (s) => s.trip_id in indexedTrips
+);
 
-const edges: Record<string, any> = {};
+reader.close();
+
+const stop_times = entries.stop_times.sort((a, b) => {
+  if (a.trip_id < b.trip_id) {
+    return -1;
+  } else if (a.trip_id > b.trip_id) {
+    return 1;
+  } else {
+    return a.stop_sequence - b.stop_sequence;
+  }
+});
+
+const edges = {};
 
 console.log("calculating edges...");
 
@@ -126,10 +152,4 @@ const output = {
   },
 };
 
-console.log(
-  entries.calendar?.sort(
-    (a, b) => parseInt(b.end_date) - parseInt(a.end_date)
-  )[0]
-);
-
 await Deno.writeTextFile("cities/boston.json", JSON.stringify(output));
diff --git a/src/components/MbtaLine.svelte b/src/components/MbtaLine.svelte
index 98b2213..73c51e0 100644
--- a/src/components/MbtaLine.svelte
+++ b/src/components/MbtaLine.svelte
@@ -1,5 +1,5 @@
 <script>
-  import agency from "../../cities/boston.json"
+  import agency from "../agency"
 
   export let name
   export let compact = false
diff --git a/src/pages/index.astro b/src/pages/index.astro
index 7e3a842..314b9cb 100644
--- a/src/pages/index.astro
+++ b/src/pages/index.astro
@@ -7,6 +7,17 @@ import screenshot from "../screenshot.png";
   <head>
     <link rel="preload" as="image" href={screenshot.src} />
     <title>The Boston Train Quiz</title>
+
+    {
+      import.meta.env.PROD && (
+        <script
+          is:inline
+          defer
+          src="https://cloud.umami.is/script.js"
+          data-website-id="3c0255b3-e006-4147-8fc3-5d87b9160e7b"
+        />
+      )
+    }
   </head>
   <body>
     <Guesser client:load />