trainvel/gtfs/fixtures/gtfs_feeds.json

@@ -31,21 +31,11 @@
     },
     {
         "model": "gtfs.gtfsfeed",
-        "pk": "FR-IDF-IDFM",
+        "pk": "FR-IDF-TN",
         "fields": {
-            "name": "Île-de-France Mobilités",
+            "name": "SNCF - Transilien",
             "country": "FR",
-            "feed_url": "https://data.iledefrance-mobilites.fr/api/v2/catalog/datasets/offre-horaires-tc-gtfs-idfm/files/a925e164271e4bca93433756d6a340d1",
-            "rt_feed_url": ""
-        }
-    },
-    {
-        "model": "gtfs.gtfsfeed",
-        "pk": "FR-GES-CTS",
-        "fields": {
-            "name": "Compagnie des Transports Strasbourgeois (CTS)",
-            "country": "FR",
-            "feed_url": "https://opendata.cts-strasbourg.eu/google_transit.zip",
+            "feed_url": "https://eu.ftp.opendatasoft.com/sncf/gtfs/transilien-gtfs.zip",
             "rt_feed_url": ""
         }
     },

trainvel/gtfs/management/commands/update_trainvel_gtfs.py

@@ -1,6 +1,6 @@
 import csv
 from datetime import datetime, timedelta
-from io import BytesIO, TextIOWrapper
+from io import BytesIO
 from zipfile import ZipFile
 from zoneinfo import ZoneInfo
 
@@ -8,8 +8,8 @@ import requests
 from django.core.management import BaseCommand
 from tqdm import tqdm
 
-from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, Stop, StopTime, \
-    Transfer, Trip, PickupType
+from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, Stop, StopTime, Transfer, Trip, \
+    PickupType
 
 
 class Command(BaseCommand):
@@ -52,18 +52,16 @@ class Command(BaseCommand):
             self.stdout.write(f"Downloading GTFS feed for {gtfs_feed}...")
             resp = requests.get(gtfs_feed.feed_url, allow_redirects=True, stream=True)
             with ZipFile(BytesIO(resp.content)) as zipfile:
-                def read_csv(filename):
-                    with zipfile.open(filename, 'r') as zf:
-                        with TextIOWrapper(zf, encoding='utf-8') as wrapper:
-                            reader = csv.DictReader(wrapper)
-                            reader.fieldnames = [field.replace('\ufeff', '').strip()
-                                                 for field in reader.fieldnames]
-                            for row in tqdm(reader, desc=filename, unit=' rows'):
-                                yield {k.strip(): v.strip() for k, v in row.items()}
+                def read_file(filename):
+                    lines = zipfile.read(filename).decode().replace('\ufeff', '').splitlines()
+                    return [line.strip() for line in lines]
 
                 agencies = []
-                for agency_dict in read_csv("agency.txt"):
+                for agency_dict in csv.DictReader(read_file("agency.txt")):
                     agency_dict: dict
+                    # if gtfs_code == "FR-EUROSTAR" \
+                    #         and agency_dict['agency_id'] != 'ES' and agency_dict['agency_id'] != 'ER':
+                    #     continue
                     agency = Agency(
                         id=f"{gtfs_code}-{agency_dict['agency_id']}",
                         name=agency_dict['agency_name'],
@@ -84,7 +82,7 @@ class Command(BaseCommand):
                     agencies.clear()
 
                 stops = []
-                for stop_dict in read_csv("stops.txt"):
+                for stop_dict in csv.DictReader(tqdm(read_file("stops.txt"), desc="Stops")):
                     stop_dict: dict
                     stop_id = stop_dict['stop_id']
                     stop_id = f"{gtfs_code}-{stop_id}"
@@ -122,15 +120,13 @@ class Command(BaseCommand):
                     stops.clear()
 
                 routes = []
-                for route_dict in read_csv("routes.txt"):
+                for route_dict in csv.DictReader(tqdm(read_file("routes.txt"), desc="Routes")):
                     route_dict: dict
                     route_id = route_dict['route_id']
                     route_id = f"{gtfs_code}-{route_id}"
-                    # Agency is optional there is only one
-                    agency_id = route_dict.get('agency_id', "") or Agency.objects.get(gtfs_feed=gtfs_feed)
                     route = Route(
                         id=route_id,
-                        agency_id=f"{gtfs_code}-{agency_id}",
+                        agency_id=f"{gtfs_code}-{route_dict['agency_id']}",
                         short_name=route_dict['route_short_name'],
                         long_name=route_dict['route_long_name'],
                         desc=route_dict.get('route_desc', ""),
@@ -162,7 +158,7 @@ class Command(BaseCommand):
                 Calendar.objects.filter(gtfs_feed=gtfs_feed).delete()
                 calendars = {}
                 if "calendar.txt" in zipfile.namelist():
-                    for calendar_dict in read_csv("calendar.txt"):
+                    for calendar_dict in csv.DictReader(tqdm(read_file("calendar.txt"), desc="Calendars")):
                         calendar_dict: dict
                         calendar = Calendar(
                             id=f"{gtfs_code}-{calendar_dict['service_id']}",
@@ -196,7 +192,7 @@ class Command(BaseCommand):
                         calendars.clear()
 
                 calendar_dates = []
-                for calendar_date_dict in read_csv("calendar_dates.txt"):
+                for calendar_date_dict in csv.DictReader(tqdm(read_file("calendar_dates.txt"), desc="Calendar dates")):
                     calendar_date_dict: dict
                     calendar_date = CalendarDate(
                         id=f"{gtfs_code}-{calendar_date_dict['service_id']}-{calendar_date_dict['date']}",
@@ -243,7 +239,7 @@ class Command(BaseCommand):
                     calendar_dates.clear()
 
                 trips = []
-                for trip_dict in read_csv("trips.txt"):
+                for trip_dict in csv.DictReader(tqdm(read_file("trips.txt"), desc="Trips")):
                     trip_dict: dict
                     trip_id = trip_dict['trip_id']
                     route_id = trip_dict['route_id']
@@ -282,7 +278,7 @@ class Command(BaseCommand):
                     trips.clear()
 
                 stop_times = []
-                for stop_time_dict in read_csv("stop_times.txt"):
+                for stop_time_dict in csv.DictReader(tqdm(read_file("stop_times.txt"), desc="Stop times")):
                     stop_time_dict: dict
 
                     stop_id = stop_time_dict['stop_id']
@@ -341,7 +337,7 @@ class Command(BaseCommand):
 
                 if "transfers.txt" in zipfile.namelist():
                     transfers = []
-                    for transfer_dict in read_csv("transfers.txt"):
+                    for transfer_dict in csv.DictReader(tqdm(read_file("transfers.txt"), desc="Transfers")):
                         transfer_dict: dict
                         from_stop_id = transfer_dict['from_stop_id']
                         to_stop_id = transfer_dict['to_stop_id']
@@ -372,7 +368,7 @@ class Command(BaseCommand):
                         transfers.clear()
 
                 if "feed_info.txt" in zipfile.namelist() and not dry_run:
-                    for feed_info_dict in read_csv("feed_info.txt"):
+                    for feed_info_dict in csv.DictReader(tqdm(read_file("feed_info.txt"), desc="Feed info")):
                         feed_info_dict: dict
                         FeedInfo.objects.update_or_create(
                             publisher_name=feed_info_dict['feed_publisher_name'],