diff --git a/.gitignore b/.gitignore index 8daaad3..322eccc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -example/ +gtfs/ node_modules/ .astro/ .DS_Store diff --git a/gtfs-importer/import.js b/gtfs-importer/import.js new file mode 100644 index 0000000..1a50afd --- /dev/null +++ b/gtfs-importer/import.js @@ -0,0 +1,194 @@ +import { TextWriter } from "jsr:@zip-js/zip-js"; +import { ZipReader } from "jsr:@zip-js/zip-js"; +import { parse } from "npm:csv-parse/sync"; +import { parse as parseStream } from "npm:csv-parse"; + +function indexRecords(records, key, processRecords = (v) => v) { + const result = {}; + + for (const record of records) { + result[record[key]] = processRecords(record); + } + + return result; +} + +async function streamRecords(entry, filter) { + const records = []; + + const parser = parseStream({ columns: true }); + + parser.on("readable", () => { + let record; + while ((record = parser.read()) !== null) { + if (filter(record)) records.push(record); + } + }); + + await entry.getData( + new WritableStream({ + write(chunk) { + parser.write(chunk); + }, + close() { + parser.end(); + }, + }) + ); + + return records; +} + +export async function importGtfs( + cityName, + { + sourceFeed, + outputFile, + allowedRouteTypes, + processStationNames = (name) => name, + serviceId, + } +) { + const reader = new ZipReader((await Deno.open(sourceFeed)).readable); + const files = await reader.getEntries(); + + const entries = {}; + + console.log("reading files"); + + await Promise.all( + files + .filter((f) => + ["routes.txt", "trips.txt", "stops.txt", "calendar.txt"].includes( + f.filename + ) + ) + .map(async (f) => { + const writer = new TextWriter(); + entries[f.filename.replace(".txt", "")] = parse( + await f.getData(writer), + { + columns: true, + } + ); + }) + ); + + console.log(`Indexing ${entries.stops?.length} stops...`); + + const indexedStops = indexRecords(entries.stops, "stop_id", (stop) => ({ + ...stop, + stop_name: processStationNames(stop.stop_name), + })); + + const allowedRoutes = entries.routes.filter((r) => + allowedRouteTypes.includes(r.route_type) + ); + + const allowedRouteIds = allowedRoutes.map((r) => r.route_id); + + console.log("Indexing trips..."); + + const indexedTrips = indexRecords( + entries.trips.filter( + (t) => + allowedRouteIds.includes(t.route_id) && + (!serviceId || t.service_id == serviceId) + ), + "trip_id" + ); + + console.log("fetching stop times..."); + + entries.stop_times = await streamRecords( + files.find((f) => f.filename == "stop_times.txt"), + (s) => s.trip_id in indexedTrips + ); + + console.log(entries.stop_times.length); + + reader.close(); + + const stop_times = entries.stop_times.sort((a, b) => { + if (a.trip_id < b.trip_id) { + return -1; + } else if (a.trip_id > b.trip_id) { + return 1; + } else { + return a.stop_sequence - b.stop_sequence; + } + }); + + const edges = {}; + + console.log("calculating edges..."); + + function actualStop(stop_id) { + return indexedStops[stop_id]?.parent_station || stop_id; + } + + for (let i = 0; i < stop_times.length - 1; i++) { + const a = stop_times[i]; + const b = stop_times[i + 1]; + + if (a.trip_id == b.trip_id) { + const trip = indexedTrips[a.trip_id]; + const route_id = trip.route_id; + const aStop = actualStop(a.stop_id); + const bStop = actualStop(b.stop_id); + const key = `${aStop}-${bStop}`; + + if (key in edges) { + edges[key].route.add(route_id); + } else { + edges[key] = { + a: aStop, + b: bStop, + route: new Set([route_id]), + headsign: trip.trip_headsign, + }; + } + } + } + + function routesForStation(station_id) { + return [ + ...new Set( + Object.values(edges) + .filter((e) => e.a == station_id || e.b == station_id) + .flatMap((e) => [...e.route]) + ), + ]; + } + + const stations = [ + ...new Set( + Object.values(edges) + .flatMap((e) => [e.a, e.b]) + .map((e) => actualStop(e)) + ), + ]; + const output = { + stations: indexRecords( + stations.map((s) => ({ + ...indexedStops[s], + routes: routesForStation(s), + })), + "stop_id" + ), + edges: Object.values(edges).map((e) => ({ ...e, route: [...e.route] })), + routes: indexRecords(allowedRoutes, "route_id"), + agency: { + city: cityName, + slug: cityName.toLowerCase().replace(/\s+/g, "-"), + }, + }; + + await Deno.writeTextFile(outputFile, JSON.stringify(output)); + + console.log("done"); +} + +export const STREETCAR = "0"; +export const SUBWAY = "1"; +export const RAIL = "2"; diff --git a/gtfs-importer/index.js b/gtfs-importer/index.js new file mode 100644 index 0000000..3ab012d --- /dev/null +++ b/gtfs-importer/index.js @@ -0,0 +1,19 @@ +import { importGtfs, STREETCAR, SUBWAY } from "./import.js"; + +await importGtfs("Boston", { + allowedRouteTypes: [STREETCAR, SUBWAY], + sourceFeed: "gtfs/mbta.zip", + outputFile: "cities/boston.json", + serviceId: "canonical", +}); + +// await importGtfs("Chicago", { +// allowedRouteTypes: ["0", "1"], +// sourceFeed: "gtfs/cta.zip", +// outputFile: "cities/chicago.json", +// processStationNames: (name) => +// name.replace( +// /\s*\((Red|Purple|Yellow|Blue|Pink|Green|Orange|Brown)\)$/, +// "" +// ), +// }); diff --git a/gtfs.js b/gtfs.js deleted file mode 100644 index 805ed29..0000000 --- a/gtfs.js +++ /dev/null @@ -1,167 +0,0 @@ -import { TextWriter } from "jsr:@zip-js/zip-js"; -import { ZipReader } from "jsr:@zip-js/zip-js"; -import { parse } from "npm:csv-parse/sync"; -import { parse as parseStream } from "npm:csv-parse"; - -function indexRecords(records, key) { - const result = {}; - - for (const record of records) { - result[record[key]] = record; - } - - return result; -} - -async function streamRecords(entry, filter) { - const records = []; - - const parser = parseStream({ columns: true }); - - parser.on("readable", () => { - let record; - while ((record = parser.read()) !== null) { - // console.log(`got record ${record.trip_id}`); - - if (filter(record)) records.push(record); - } - }); - - await entry.getData( - new WritableStream({ - write(chunk) { - parser.write(chunk); - }, - close() { - parser.end(); - }, - }) - ); - - return records; -} - -const reader = new ZipReader((await Deno.open(Deno.args[0])).readable); - -const files = await reader.getEntries(); - -const wantedFiles = ["routes", "trips", "stops", "calendar"]; - -const entries = {}; - -console.log("reading files"); - -await Promise.all( - files - .filter((f) => { - return wantedFiles.includes(f.filename.replace(".txt", "")); - }) - .map(async (f) => { - const writer = new TextWriter(); - entries[f.filename.replace(".txt", "")] = parse(await f.getData(writer), { - columns: true, - }); - }) -); - -console.log(`Indexing ${entries.stops?.length} stops...`); - -function actualStop(stop_id) { - return indexedStops[stop_id]?.parent_station || stop_id; -} - -const indexedStops = indexRecords(entries.stops, "stop_id"); - -const metroRoutes = entries.routes.filter( - (r) => r.route_type == 1 || r.route_type == 0 -); - -const metroRouteIds = metroRoutes.map((r) => r.route_id); - -console.log(`Indexing trips...`); - -const trips = entries.trips.filter( - (t) => t.service_id == "canonical" && metroRouteIds.includes(t.route_id) -); -const indexedTrips = indexRecords(trips, "trip_id"); - -console.log("fetching stop times"); - -entries.stop_times = await streamRecords( - files.find((f) => f.filename == "stop_times.txt"), - (s) => s.trip_id in indexedTrips -); - -reader.close(); - -const stop_times = entries.stop_times.sort((a, b) => { - if (a.trip_id < b.trip_id) { - return -1; - } else if (a.trip_id > b.trip_id) { - return 1; - } else { - return a.stop_sequence - b.stop_sequence; - } -}); - -const edges = {}; - -console.log("calculating edges..."); - -for (let i = 0; i < stop_times.length - 1; i++) { - const a = stop_times[i]; - const b = stop_times[i + 1]; - - if (a.trip_id == b.trip_id) { - const trip = indexedTrips[a.trip_id]; - const route_id = trip.route_id; - const aStop = actualStop(a.stop_id); - const bStop = actualStop(b.stop_id); - const key = `${aStop}-${bStop}`; - - if (key in edges) { - edges[key].route.add(route_id); - } else { - edges[key] = { - a: aStop, - b: bStop, - route: new Set([route_id]), - headsign: trip.trip_headsign, - }; - } - } -} - -function routesForStation(station_id) { - return [ - ...new Set( - Object.values(edges) - .filter((e) => e.a == station_id || e.b == station_id) - .flatMap((e) => [...e.route]) - ), - ]; -} - -const stations = [ - ...new Set( - Object.values(edges) - .flatMap((e) => [e.a, e.b]) - .map((e) => actualStop(e)) - ), -]; -const output = { - stations: indexRecords( - stations.map((s) => ({ ...indexedStops[s], routes: routesForStation(s) })), - "stop_id" - ), - edges: Object.values(edges).map((e) => ({ ...e, route: [...e.route] })), - routes: indexRecords(metroRoutes, "route_id"), - agency: { - city: "Boston", - slug: "boston", - }, -}; - -await Deno.writeTextFile("cities/boston.json", JSON.stringify(output)); - -console.log("done");