diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index 37c7e82..629b5f7 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -1,458 +1,8 @@ #!/usr/env/bin python3 - -import json -import datetime -import re -import pytz - -import psycopg2 as pg -import psycopg2.extras as pge +import subprocess from django.core.management.base import BaseCommand from django.core.management import call_command -from django.db import transaction -from django.core.exceptions import ValidationError -from django.utils.timezone import make_aware -from django.db import IntegrityError -from django.contrib.auth.models import User - -from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction -from note.models import Note -from note.models import Alias -from note.models import ( - TemplateCategory, - TransactionTemplate, - Transaction, - RecurrentTransaction, - MembershipTransaction, - SpecialTransaction, -) -from member.models import Club, Membership -from treasury.models import RemittanceType, Remittance, SpecialTransactionProxy - -""" -Script d'import de la nk15: -TODO: import transactions -TODO: import adhesion -TODO: import activite -TODO: import ... - -""" -M_DURATION = 396 -M_START = datetime.date(2019, 8, 31) -M_END = datetime.date(2020, 9, 30) - -MAP_IDBDE = { - -4: 2, # Carte Bancaire - -3: 4, # Virement - -2: 1, # Especes - -1: 3, # Chèque - 0: 5, # BDE -} - -MAP_IDACTIVITY = {} -MAP_NAMEACTIVITY = {} -MAP_NAMEGUEST = {} -MAP_IDSPECIALTRANSACTION = {} - - -def update_line(n, total, content): - n = str(n) - total = str(total) - n.rjust(len(total)) - print(f"\r ({n}/{total}) {content:10.10}", end="") - - -@transaction.atomic -def import_comptes(cur): - cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;") - pkclub = 3 - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["pseudo"]) - if row["type"] == "personne": - # sanitize password - if row["passwd"] != "*|*" and not row["deleted"]: - passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]]) - else: - passwd_nk15 = '' - try: - obj_dict = { - "username": row["pseudo"], - "password": passwd_nk15, - "first_name": row["nom"], - "last_name": row["prenom"], - "email": row["mail"], - "is_active": True, # temporary - } - - user = User.objects.create(**obj_dict) - profile = user.profile - profile.phone_number = row['tel'] - profile.address = row['adresse'] - profile.paid = row['normalien'] - profile.registration_valid = True - profile.email_confirmed = True - user.save() - profile.save() - # sanitize duplicate aliases (nk12) - except ValidationError as e: - if e.code == 'same_alias': - user.username = row["pseudo"] + str(row["idbde"]) - user.save() - else: - raise e - # profile and note created via signal. - - note = user.note - date = row.get("last_negatif", None) - if date is not None: - note.last_negative = make_aware(date) - note.balance = row["solde"] - note.save() - else: # club - obj_dict = { - "pk": pkclub, - "name": row["pseudo"], - "email": row["mail"], - "membership_duration": M_DURATION, - "membership_start": M_START, - "membership_end": M_END, - "membership_fee_paid": 0, - "membership_fee_unpaid": 0, - } - club, c = Club.objects.get_or_create(**obj_dict) - pkclub += 1 - note = club.note - note.balance = row["solde"] - club.save() - note.save() - - MAP_IDBDE[row["idbde"]] = note.note_ptr_id - - -@transaction.atomic -def import_boutons(cur): - cur.execute("SELECT * FROM boutons;") - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["label"]) - cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"]) - if created: - cat.save() - obj_dict = { - "pk": row["id"], - "name": row["label"], - "amount": row["montant"], - "destination_id": MAP_IDBDE[row["destinataire"]], - "category": cat, - "display": row["affiche"], - "description": row["description"], - } - try: - with transaction.atomic(): # required for error management - button = TransactionTemplate.objects.create(**obj_dict) - except IntegrityError as e: - # button with the same name is not possible in NK20. - if "unique" in e.args[0]: - qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name') - note_name = qs[0]["name"] - # rename button name - obj_dict["name"] = f"{obj_dict_name['name']} {note_name}" - button = TransactionTemplate.objects.create(**obj_dict) - else: - raise e - button.save() - - -@transaction.atomic -def import_transaction(cur): - idmin = 58770 - bde = Club.objects.get(name="BDE") - kfet = Club.objects.get(name="Kfet") - cur.execute( - "SELECT t.date AS transac_date, t.type, t.emetteur,\ - t.destinataire,t.quantite, t.montant, t.description,\ - t.valide, t.cantinvalidate, t.categorie, \ - a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ - FROM transactions AS t \ - LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ - WHERE t.id> {} \ - ORDER BY t.id;".format(idmin) - ) - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["description"]) - try: - date = make_aware(row["transac_date"]) - except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): - date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) - - # standart transaction object - obj_dict = { - # "pk": row["id"], - "destination_id": MAP_IDBDE[row["destinataire"]], - "source_id": MAP_IDBDE[row["emetteur"]], - "created_at": date, - "amount": row["montant"], - "quantity": row["quantite"], - "reason": row["description"], - "valid": row["valide"], - } - ttype = row["type"] - if ttype == "don" or ttype == "transfert": - Transaction.objects.create(**obj_dict) - elif ttype == "bouton": - cat_name = row["categorie"] - if cat_name is None: - cat_name = 'None' - cat, created = TemplateCategory.objects.get_or_create(name=cat_name) - if created: - cat.save() - obj_dict["category"] = cat - RecurrentTransaction.objects.create(**obj_dict) - elif ttype == "crédit" or ttype == "retrait": - field_id = "source_id" if ttype == "crédit" else "destination_id" - if "espèce" in row["description"]: - obj_dict[field_id] = 1 - elif "carte" in row["description"]: - obj_dict[field_id] = 2 - elif "cheques" in row["description"]: - obj_dict[field_id] = 3 - elif "virement" in row["description"]: - obj_dict[field_id] = 4 - pk = max(row["destinataire"], row["emetteur"]) - actor = Note.objects.get(id=MAP_IDBDE[pk]) - # custom fields of SpecialTransaction - if actor.__class__.__name__ == "NoteUser": - obj_dict["first_name"] = actor.user.first_name - obj_dict["last_name"] = actor.user.last_name - elif actor.__class__.__name__ == "NoteClub": - obj_dict["first_name"] = actor.club.name - obj_dict["last_name"] = actor.club.name - else: - raise Exception("Badly formatted Special Transaction You should'nt be there.") - tr = SpecialTransaction.objects.create(**obj_dict) - if "cheques" in row["description"]: - MAP_IDSPECIALTRANSACTION[row["id"]] = tr - - elif ttype == "adhésion": - montant = row["montant"] - - # Create Double membership to Kfet and Bde - # sometimes montant = 0, fees are modified accordingly. - bde_dict = { - "user": MAP_IDBDE[row["idbde"]], - "club": bde, - "date_start": row["date"].date(), # Only date, not time - "fee": min(500, montant) - } - kfet_dict = { - "user": MAP_IDBDE[row["idbde"]], - "club": kfet, - "date_start": row["date"].date(), # Only date, not time - "fee": max(montant - 500, 0), - } - - if row["valide"]: - with transaction.atomic(): - # membership save triggers MembershipTransaction creation - bde_membership = Membership.objects.get_or_create(**bde_dict) - kfet_membership = Membership.objects.get_or_create(**kfet_dict) - bde_membership.transaction.created_at = row["transac_date"] - bde_membership.transaction.description = row["description"] - bde_membership.transaction.save() - kfet_membership.transaction.created_at = row["transac_date"] - kfet_membership.transaction.description = row["description"] + "(Kfet)" - kfet_membership.transaction.save() - else: - # don't create membership - MembershipTransaction.objects.create(**obj_dict) - elif ttype == "invitation": - m = re.search(r"Invitation (.*?) \((.*?)\)", row["description"]) - if m is None: - raise IntegrityError(f"Invitation is not well formated: {row['description']} (must be 'Invitation ACTIVITY_NAME (NAME)')") - - activity_name = m.group(1) - guest_name = m.group(2) - - if activity_name not in MAP_NAMEACTIVITY: - raise IntegrityError(f"Activity {activity_name} is not found") - activity = MAP_NAMEACTIVITY[activity_name] - - if guest_name not in MAP_NAMEGUEST: - raise IntegrityError(f"Guest {guest_name} is not found") - - guest = None - for g in MAP_NAMEGUEST[guest_name]: - if g.activity.pk == activity.pk: - guest = g - break - if guest is None: - raise IntegrityError("Guest {guest_name} didn't go to the activity {activity_name}") - - obj_dict["guest"] = guest - - GuestTransaction.objects.get_or_create(**obj_dict) - else: - print("other type not supported yet:", ttype) - - -@transaction.atomic -def import_aliases(cur): - cur.execute("SELECT * FROM aliases ORDER by id") - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["alias"]) - alias_name = row["alias"] - alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name - obj_dict = { - "note_id": MAP_IDBDE[row["idbde"]], - "name": alias_name_good, - "normalized_name": Alias.normalize(alias_name_good), - } - try: - with transaction.atomic(): - alias, created = Alias.objects.get_or_create(**obj_dict) - - except IntegrityError as e: - if "unique" in e.args[0]: - continue - else: - raise e - alias.save() - - -@transaction.atomic -def import_activities(cur): - cur.execute("SELECT * FROM activites ORDER by id") - n = cur.rowcount - activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually - kfet = Club.objects.get(name="Kfet") - for idx, row in enumerate(cur): - update_line(idx, n, row["alias"]) - organizer = Club.objects.filter(name=row["signature"]) - if organizer.exists(): - # Try to find the club that organizes the activity. If not found, assume it's Kfet (fix manually) - organizer = organizer.get() - else: - organizer = kfet - obj_dict = { - "name": row["titre"], - "description": row["description"], - "activity_type": activity_type, # By default Pot - "creater": MAP_IDBDE[row["responsable"]], - "organizer": organizer, - "attendees_club": kfet, # Maybe fix manually - "date_start": row["debut"], - "date_end": row["fin"], - "valid": row["validepar"] is not None, - "open": row["open"], # Should always be False - } - # WARNING: Fields lieu, liste, listeimprimee are missing - try: - with transaction.atomic(): - activity = Activity.objects.get_or_create(**obj_dict)[0] - MAP_IDACTIVITY[row["id"]] = activity - MAP_NAMEACTIVITY[activity.name] = activity - except IntegrityError as e: - raise e - - -@transaction.atomic -def import_activity_entries(cur): - map_idguests = {} - - cur.execute("SELECT * FROM invites ORDER by id") - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["nom"] + " " + row["prenom"]) - obj_dict = { - "activity": MAP_IDACTIVITY[row["activity"]], - "last_name": row["nom"], - "first_name": row["prenom"], - "inviter": MAP_IDBDE[row["responsable"]], - } - try: - with transaction.atomic(): - guest = Guest.objects.get_or_create(**obj_dict)[0] - map_idguests.setdefault(row["responsable"], []) - map_idguests[row["id"]].append(guest) - guest_name = guest.first_name + " " + guest.last_name - MAP_NAMEGUEST.setdefault(guest_name, []) - MAP_NAMEGUEST[guest_name].append(guest) - except IntegrityError as e: - raise e - - cur.execute("SELECT * FROM entree_activites ORDER by id") - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["nom"] + " " + row["prenom"]) - activity = MAP_IDACTIVITY[row["activity"]] - guest = None - if row["est_invite"]: - for g in map_idguests[row["id"]]: - if g.activity.pk == activity.pk: - guest = g - break - if not guest: - raise IntegrityError("Guest was not found: " + str(row)) - obj_dict = { - "activity": activity, - "time": row["heure_entree"], - "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]], - "guest": guest, - } - try: - with transaction.atomic(): - Entry.objects.get_or_create(**obj_dict) - except IntegrityError as e: - raise e - - -@transaction.atomic -def import_remittances(cur): - cur.execute("SELECT * FROM remises ORDER by id") - map_idremittance = {} - n = cur.rowcount - check_type = RemittanceType.objects.get(note__name="Chèque") - for idx, row in enumerate(cur): - update_line(idx, n, row["date"]) - obj_dict = { - "date": row["date"][10:], - "remittance_type": check_type, - "comment": row["commentaire"], - "closed": row["close"], - } - try: - with transaction.atomic(): - remittance = Remittance.objects.get_or_create(**obj_dict) - map_idremittance[row["id"]] = remittance - except IntegrityError as e: - raise e - - print("remittances are imported") - print("imported checks") - - cur.execute("SELECT * FROM cheques ORDER by id") - n = cur.rowcount - for idx, row in enumerate(cur): - update_line(idx, n, row["date"]) - obj_dict = { - "date": row["date"][10:], - "remittance_type": check_type, - "comment": row["commentaire"], - "closed": row["close"], - } - tr = MAP_IDSPECIALTRANSACTION[row["idtransaction"]] - proxy = SpecialTransactionProxy.objects.get_or_create(transaction=tr) - proxy.remittance = map_idremittance[row["idremise"]] - try: - with transaction.atomic(): - proxy.save() - except IntegrityError as e: - raise e - class Command(BaseCommand): """ @@ -460,60 +10,9 @@ class Command(BaseCommand): Need to be run by a user with a registered role in postgres for the database nk15. """ - def print_success(self, to_print): - return self.stdout.write(self.style.SUCCESS(to_print)) - - def add_arguments(self, parser): - parser.add_argument('-c', '--comptes', action='store_true', help="import accounts") - parser.add_argument('-b', '--boutons', action='store_true', help="import boutons") - parser.add_argument('-t', '--transactions', action='store_true', help="import transaction") - parser.add_argument('-al', '--aliases', action='store_true', help="import aliases") - parser.add_argument('-ac', '--activities', action='store_true', help="import activities") - parser.add_argument('-r', '--remittances', action='store_true', help="import check remittances") - parser.add_argument('-s', '--save', action='store', help="save mapping of idbde") - parser.add_argument('-m', '--map', action='store', help="import mapping of idbde") - parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name') - parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner') - def handle(self, *args, **kwargs): - global MAP_IDBDE - nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] - # connecting to nk15 database - conn = pg.connect(database=nk15db, user=nk15user) - cur = conn.cursor(cursor_factory=pge.DictCursor) - - if kwargs["comptes"]: - # reset database. - call_command("migrate") - call_command("loaddata", "initial") - self.print_success("reset nk20 database\n") - import_comptes(cur) - self.print_success("comptes table imported") - elif kwargs["map"]: - filename = kwargs["map"] - with open(filename, 'r') as fp: - MAP_IDBDE = json.load(fp) - MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()} - if kwargs["save"]: - filename = kwargs["save"] - with open(filename, 'w') as fp: - json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2) - - # /!\ need a prober MAP_IDBDE - if kwargs["boutons"]: - import_boutons(cur) - self.print_success("boutons table imported\n") - if kwargs["activities"]: - import_activities(cur) - self.print_success("activities imported\n") - import_activity_entries(cur) - self.print_success("activity entries imported\n") - if kwargs["aliases"]: - import_aliases(cur) - self.print_success("aliases imported\n") - if kwargs["transactions"]: - import_transaction(cur) - self.print_success("transaction imported\n") - if kwargs["remittances"]: - import_remittances(cur) - self.print_success("remittances imported\n") + subprocess.call("./apps/scripts/shell/tabularasa") + call_command('import_account', alias=True, chunk=1000, save = "map.json") + call_command('import_activities', chunk=100, map="map.json") + call_command('import_transaction', buttons=True, map="map.json") +#