send summary script

Remove useless tee, use "mkdir -p" and use .sql.gz.

README.md

@@ -1,63 +1,3 @@
 # Script de la NoteKfet 2020
 
-## Commandes Django
+La documentation est disponible sur <https://note.crans.org/doc/scripts/>.
-
-> les commandes sont documentées:
-> `./manage.py command --help`
-
-- `import_nk15` :
-
-    Importe un dump de la NoteKfet 2015. 
-
-- `make_su [--STAFF|-s] [--SUPER|-S]` :
-
-    Rend actifs les pseudos renseignés.
-  
-    * Si `--STAFF` ou `-s` est renseigné, donne en plus le statut d'équipe aux pseudos renseignés, 
-    permettant l'accès à l'interface admin.
-    * Si `--SUPER` ou `-S` est renseigné, donne en plus le statut de super-utilisateur aux pseudos renseignés,
-    octroyant tous les droits sur la plateforme.
-
-- `wei_algorithm` :
-
-    Lance l'algorithme de répartition des 1A au dernier WEI. Cela a pour effet de suggérer un bus pour tous les 1A
-    inscrits au dernier WEI en fonction des données rentrées dans le sondage, la validation se faisant ensuite
-    manuellement via l'interface Web.
-
-- `extract_ml_registrations --type {members, clubs, events, art, sport} [--year|-y YEAR]` :
-
-    Récupère la liste des adresses mail à inscrire à une liste de diffusion donnée.
-    
-    * `members` : Liste des adresses mail des utilisateurs ayant une adhésion BDE (et non Kfet) active.
-    * `clubs` : Liste des adresses mail de contact de tous les clubs BDE enregistrés.
-    * `events` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                 la liste de diffusion des événements du BDE.
-    * `art` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-              la liste de diffusion concertnant les actualités artistiques du BDA.
-    * `sport` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                la liste de diffusion concertnant les actualités sportives du BDS.
-    
-    Le champ `--year` est optionnel : il permet de choisir l'année du WEI en question (pour les trois dernières 
-    options). Si non renseigné, il s'agit du dernier WEI.
-    
-    Par défaut, si `--type` est non renseigné, la liste des adhérents BDE est renvoyée.
-
-- `extract_wei_registrations [--year|-y YEAR] [--bus|-b BUS] [--team|-t TEAM] [--sep SEP]` :
-
-    Récupère la liste des inscriptions au WEI et l'exporte au format CSV. Arguments possibles, optionnels :
-    
-    * `--year YEAR` : sélectionne l'année du WEI. Par défaut, il s'agit du dernier WEI ayant eu lieu.
-    * `--bus BUS` : filtre par bus, en récupérant uniquement les inscriptions sur un bus. Par défaut, on affiche
-    tous les bus.
-    * `--team TEAM` : filtre par équipe, en récupérant uniquement les inscriptions sur une équipe. Par défaut, on
-    affiche toutes les équipes. Entrer `"none"` filtre les inscriptions sans équipe (chefs de bus, ...)
-    * `--sep` : définit le caractère de séparation des colonnes du fichier CSV. Par défaut, il s'agit du caractère `|`.
-    Merci de ne pas rentrer plus d'un caractère.
-    
-
-## Shell
-
-- Tabula rasa :
-```shell script
-sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"
-```

__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 default_app_config = 'scripts.apps.ScriptsConfig'

apps.py

@@ -1,13 +1,8 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.apps import AppConfig
-from django.core.signals import got_request_exception
 
 
 class ScriptsConfig(AppConfig):
     name = 'scripts'
-
-    def ready(self):
-        from . import signals
-        got_request_exception.connect(signals.send_mail_on_exception)

management/commands/_import_utils.py

@@ -1,13 +1,13 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import json
 import time
 from collections import defaultdict
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import transaction
-
 from polymorphic.models import PolymorphicModel
 
 
@@ -15,11 +15,12 @@ def timed(method):
     """"
     A simple decorator to measure time elapsed in class function (hence the args[0])
     """
+
     def _timed(*args, **kw):
         ts = time.time()
         result = method(*args, **kw)
         te = time.time()
-        args[0].print_success(f"{method.__name__} executed ({te-ts:.2f}s)")
+        args[0].print_success(f"\n    {method.__name__} executed ({te-ts:.2f}s)")
         return result
 
     return _timed
@@ -44,14 +45,14 @@ class ImportCommand(BaseCommand):
         n = str(n)
         total = str(total)
         n.rjust(len(total))
-        print(f"\r ({n}/{total}) {content:10.10}", end="")
+        print(f"\r ({n}/{total}) {content:16.16}", end="")
 
     def create_parser(self, prog_name, subcommand, **kwargs):
         parser = super().create_parser(prog_name, subcommand, **kwargs)
         parser.add_argument('--nk15db', action='store', default='nk15', help='NK15 database name')
         parser.add_argument('--nk15user', action='store', default='nk15_user', help='NK15 database owner')
-        parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
+        parser.add_argument('-s', '--save', default='map.json', action='store', help="save mapping of idbde")
-        parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
+        parser.add_argument('-m', '--map', default='map.json', action='store', help="import mapping of idbde")
         parser.add_argument('-c', '--chunk', type=int, default=100, help="chunk size for bulk_create")
         return parser
 

management/commands/anonymize_data.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2018-2023 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    """
+    Command to protect sensitive data during the beta phase or after WEI.
+    Phone number, email address, postal address, first and last name,
+    IP addresses, health issues, gender and birth date are removed.
+    """
+    def add_arguments(self, parser):
+        parser.add_argument('--force', '-f', action='store_true', help="Actually anonymize data.")
+        parser.add_argument('--type', '-t', choices=["all", "wei", "user"], default="",
+                            help='Select the type of data to anonyze (default None)')
+
+    def handle(self, *args, **options):
+        if not options['force']:
+            if options['type'] == "all":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset ALL personal data with "
+                              "sample data. Don't use  in production! If you know what you are doing, please "
+                              "add --force option.")
+            elif options['type'] == "wei":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset WEI personal data with "
+                              "sample data. Use it in production only after WEI. If you know what you are doing,"
+                              "please add --force option.")
+            elif options['type'] == "user":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data "
+                              "visible by user (not admin or trez BDE) with sample data. Don't use  in "
+                              "production! If you know what you are doing, please add --force option.")
+            else:
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data with "
+                              "sample data. Don't use  in production ('wei' can be use in production after "
+                              "the WEI)! If you know what you are doing, please choose a type.")
+            exit(1)
+
+        cur = connection.cursor()
+        if options['type'] in ("all","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize profile, user club and guest data")
+            cur.execute("UPDATE member_profile SET "
+                    "phone_number = '0123456789', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE auth_user SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme', "
+                    "email = 'anonymous@example.com';")
+            cur.execute("UPDATE member_club SET "
+                    "email = 'anonymous@example.com';")
+            cur.execute("UPDATE activity_guest SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+
+        if options['type'] in ("all","wei","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize WEI data")
+            cur.execute("UPDATE wei_weiregistration SET "
+                    "birth_date = '1998-01-08', "
+                    "emergency_contact_name = 'Anne Onyme', "
+                    "emergency_contact_phone = '0123456789', "
+                    "gender = 'nonbinary', "
+                    "health_issues = 'Tout va bien';")
+
+        if options['type'] == "all":
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize invoice, special transaction, log, mailer and oauth data")
+            cur.execute("UPDATE treasury_invoice SET "
+                    "name = 'Anne Onyme', "
+                    "object = 'Rends nous riches', "
+                    "description = 'Donne nous plein de sous', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE treasury_product SET "
+                    "designation = 'un truc inutile';")
+            cur.execute("UPDATE note_specialtransaction SET "
+                    "bank = 'le matelas', "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+            cur.execute("UPDATE logs_changelog SET "
+                    "ip = '127.0.0.1', "
+                    "data = 'new data', "
+                    "previous = 'old data';")
+            cur.execute("UPDATE mailer_messagelog SET "
+                    "log_message = 'log message', "
+                    "message_data = 'message data';")
+            cur.execute("UPDATE mailer_dontsendentry SET "
+                    "to_address = 'anonymous@example.com';")
+            cur.execute("UPDATE oauth2_provider_application SET "
+                    "name = 'external app', "
+                    "redirect_uris = 'http://external.app', "
+                    "client_secret = 'abcdefghijklmnopqrstuvwxyz';")
+        cur.close()

management/commands/check_consistency.py

@@ -0,0 +1,54 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management import BaseCommand
+from django.db.models import Sum, F
+
+from note.models import Note, Transaction
+from note.templatetags.pretty_money import pretty_money
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--sum-all', '-s', action='store_true', help='Check if the global sum is equal to zero')
+        parser.add_argument('--check-all', '-a', action='store_true', help='Check all notes')
+        parser.add_argument('--check', '-c', type=int, nargs='+', help='Select note ids')
+        parser.add_argument('--fix', '-f', action='store_true', help='Fix note balances')
+
+    def handle(self, *args, **options):
+        error = False
+        err_log = ""
+
+        if options["sum_all"]:
+            s = Note.objects.aggregate(Sum("balance"))["balance__sum"]
+            if s:
+                self.stderr.write(self.style.NOTICE("LA SOMME DES NOTES NE VAUT PAS ZÉRO : " + pretty_money(s)))
+                error = True
+            else:
+                if options["verbosity"] > 0:
+                    self.stdout.write(self.style.SUCCESS("La somme des notes vaut bien zéro."))
+
+        notes = Note.objects.none()
+        if options["check_all"]:
+            notes = Note.objects.all()
+        elif options["check"]:
+            notes = Note.objects.filter(pk__in=options["check"])
+
+        for note in notes:
+            balance = note.balance
+            incoming = Transaction.objects.filter(valid=True, destination=note)\
+                .annotate(total=F("quantity") * F("amount")).aggregate(Sum("total"))["total__sum"] or 0
+            outcoming = Transaction.objects.filter(valid=True, source=note)\
+                .annotate(total=F("quantity") * F("amount")).aggregate(Sum("total"))["total__sum"] or 0
+            calculated_balance = incoming - outcoming
+            if calculated_balance != balance:
+                self.stderr.write(self.style.NOTICE(f"LA SOMME DES TRANSACTIONS DE LA NOTE {note} NE CORRESPOND PAS "
+                                                    "AVEC LE MONTANT RÉEL"))
+                self.stderr.write(self.style.NOTICE(f"Attendu : {pretty_money(balance)}, "
+                                                    f"calculé : {pretty_money(calculated_balance)}"))
+                if options["fix"]:
+                    note.balance = calculated_balance
+                    note.save()
+                error = True
+
+        exit(1 if error else 0)

management/commands/compilejsmessages.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import translation
+from django.views.i18n import JavaScriptCatalog
+
+
+class Command(BaseCommand):
+    """
+    Generate Javascript translation files
+    """
+
+    def handle(self, *args, **kwargs):
+        for code, _ in settings.LANGUAGES:
+            if code == settings.LANGUAGE_CODE:
+                continue
+            if kwargs["verbosity"] > 0:
+                self.stdout.write(f"Generate {code} javascript localization file")
+            with translation.override(code):
+                resp = JavaScriptCatalog().get(None, packages="member+note")
+                if not os.path.isdir(settings.STATIC_ROOT + "/js/jsi18n"):
+                    os.makedirs(settings.STATIC_ROOT + "/js/jsi18n")
+                with open(settings.STATIC_ROOT + f"/js/jsi18n/{code}.js", "wb") as f:
+                    f.write(resp.content)

management/commands/extract_ml_registrations.py

@@ -0,0 +1,63 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date
+
+from django.contrib.auth.models import User
+from django.core.management import BaseCommand
+from member.models import Club, Membership
+
+
+class Command(BaseCommand):
+    help = "Get mailing list registrations from the last wei. " \
+           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -t {fr, en} -y {0, 1, ...}. " \
+           "You can write this into a file with a pipe, then paste the document into your mail manager."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--type', '-t', choices=["members", "clubs", "events", "art", "sport"], default="members",
+                            help='Select the type of the mailing list (default members)')
+        parser.add_argument('--lang', '-l', type=str, choices=['fr', 'en'], default='fr',
+                            help='Select the registred users of the ML of the given language. Useful only for the '
+                                 'events mailing list.')
+        parser.add_argument('--years', '-y', type=int, default=0, 
+                            help='Select the cumulative registred users of a membership from years ago. 0 means the current users')
+
+    def handle(self, *args, **options):
+        # TODO: Improve the mailing list extraction system, and link it automatically with Mailman.
+
+        if options['verbosity'] == 0:
+            # This is useless, but this what the user asked.
+            return
+
+        if options["type"] == "members":
+            today_date = date.today()
+            selected_date = date(today_date.year - options["years"], today_date.month, today_date.day)
+            for membership in Membership.objects.filter(
+                club__name="BDE",
+                date_start__lte=today_date,
+                date_end__gte=selected_date,
+            ).all():
+                self.stdout.write(membership.user.email)
+            return
+
+        if options["type"] == "clubs":
+            for club in Club.objects.all():
+                self.stdout.write(club.email)
+            return
+
+        # Get the list of mails that want to be registered to the events mailing list.
+        # Don't filter to valid members, old members can receive these mails as long as they want.
+        if options["type"] == "events":
+            for user in User.objects.filter(profile__ml_events_registration=options["lang"]).all():
+                self.stdout.write(user.email)
+            return
+
+        if options["type"] == "art":
+            for user in User.objects.filter(profile__ml_art_registration=True).all():
+                self.stdout.write(user.email)
+            return
+
+        if options["type"] == "sport":
+            for user in User.objects.filter(profile__ml_sport_registration=True).all():
+                self.stdout.write(user.email)
+            return

management/commands/force_delete_user.py

@@ -0,0 +1,176 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import getpass
+from time import sleep
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.mail import mail_admins
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.db.models import Q
+from django.test import override_settings
+from note.models import Alias, Transaction
+
+
+class Command(BaseCommand):
+    """
+    This script is used to force delete a user.
+    THIS IS ONLY ATTENDED TO BE USED TO DELETE FAKE ACCOUNTS THAT
+    WERE VALIDATED BY ERRORS. Please use data anonymization if you
+    want to block the account of a user.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument('user', type=str, nargs='+', help="User id to delete.")
+        parser.add_argument('--force', '-f', action='store_true',
+                            help="Force the script to have low verbosity.")
+        parser.add_argument('--doit', '-d', action='store_true',
+                            help="Don't ask for a final confirmation and commit modification. "
+                                 "This option should really be used carefully.")
+
+    def handle(self, *args, **kwargs):
+        force = kwargs['force']
+
+        if not force:
+            self.stdout.write(self.style.WARNING("This is a dangerous script. "
+                                                 "Please use --force to indicate that you known what you are doing. "
+                                                 "Nothing will be deleted yet."))
+            sleep(5)
+
+        # We need to know who to blame.
+        qs = User.objects.filter(note__alias__normalized_name=Alias.normalize(getpass.getuser()))
+        if not qs.exists():
+            self.stderr.write(self.style.ERROR("I don't know who you are. Please add your linux id as an alias of "
+                                               "your own account."))
+            exit(2)
+        executor = qs.get()
+
+        deleted_users = []
+        deleted = []
+
+        # Don't send mails during the process
+        with override_settings(EMAIL_BACKEND='django.core.mail.backends.dummy.EmailBackend'):
+            for user_id in kwargs['user']:
+                if user_id.isnumeric():
+                    qs = User.objects.filter(pk=int(user_id))
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
+                        continue
+                    user = qs.get()
+                else:
+                    qs = Alias.objects.filter(normalized_name=Alias.normalize(user_id), note__noteuser__isnull=False)
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
+                        continue
+                    user = qs.get().note.user
+
+                with transaction.atomic():
+                    local_deleted = []
+
+                    # Unlock note to enable modifications
+                    if force and not user.note.is_active:
+                        user.note.is_active = True
+                        user.note.save()
+
+                    # Deleting transactions
+                    transactions = Transaction.objects.filter(Q(source=user.note) | Q(destination=user.note)).all()
+                    local_deleted += list(transactions)
+                    for tr in transactions:
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Removing {tr}…")
+                        if force:
+                            tr.delete()
+
+                    # Deleting memberships
+                    memberships = user.memberships.all()
+                    local_deleted += list(memberships)
+                    if kwargs['verbosity'] >= 1:
+                        for membership in memberships:
+                            self.stdout.write(f"Removing {membership}…")
+                    if force:
+                        memberships.delete()
+
+                    # Deleting aliases
+                    alias_set = user.note.alias.all()
+                    local_deleted += list(alias_set)
+                    if kwargs['verbosity'] >= 1:
+                        for alias in alias_set:
+                            self.stdout.write(f"Removing alias {alias}…")
+                    if force:
+                        alias_set.delete()
+
+                    if 'activity' in settings.INSTALLED_APPS:
+                        from activity.models import Guest, Entry
+
+                        # Deleting activity entries
+                        entries = Entry.objects.filter(Q(note=user.note) | Q(guest__inviter=user.note)).all()
+                        local_deleted += list(entries)
+                        if kwargs['verbosity'] >= 1:
+                            for entry in entries:
+                                self.stdout.write(f"Removing {entry}…")
+                        if force:
+                            entries.delete()
+
+                        # Deleting invited guests
+                        guests = Guest.objects.filter(inviter=user.note).all()
+                        local_deleted += list(guests)
+                        if kwargs['verbosity'] >= 1:
+                            for guest in guests:
+                                self.stdout.write(f"Removing guest {guest}…")
+                        if force:
+                            guests.delete()
+
+                    if 'treasury' in settings.INSTALLED_APPS:
+                        from treasury.models import SogeCredit
+
+                        # Deleting soge credit
+                        credits = SogeCredit.objects.filter(user=user).all()
+                        local_deleted += list(credits)
+                        if kwargs['verbosity'] >= 1:
+                            for credit in credits:
+                                self.stdout.write(f"Removing {credit}…")
+                        if force:
+                            credits.delete()
+
+                    # Deleting note
+                    local_deleted.append(user.note)
+                    if force:
+                        user.note.delete()
+
+                    if 'logs' in settings.INSTALLED_APPS:
+                        from logs.models import Changelog
+
+                        # Replace log executors by the runner of the script
+                        Changelog.objects.filter(user=user).update(user=executor)
+
+                    # Deleting profile
+                    local_deleted.append(user.profile)
+                    if force:
+                        user.profile.delete()
+
+                    # Finally deleting user
+                    if force:
+                        user.delete()
+                    local_deleted.append(user)
+
+                    # This script should really not be used.
+                    if not kwargs['doit'] and not input('You are about to delete real user data. '
+                                                        'Are you really sure that it is what you want? [y/N] ')\
+                            .lower().startswith('y'):
+                        self.stdout.write(self.style.ERROR("Aborted."))
+                        exit(1)
+
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(self.style.SUCCESS(f"User {user} deleted."))
+                        deleted_users.append(user)
+                        deleted += local_deleted
+
+        if deleted_users:
+            message = f"Les utilisateur⋅rices {deleted_users} ont été supprimé⋅es par {executor}.\n\n"
+            message += "Ont été supprimés en conséquence les objets suivants :\n\n"
+            for obj in deleted:
+                message += f"{repr(obj)} (pk: {obj.pk})\n"
+            if force and kwargs['doit']:
+                mail_admins("Utilisateur⋅rices supprimés", message)

management/commands/import_account.py

@@ -1,24 +1,22 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
-import psycopg2 as pg
-import psycopg2.extras as pge
 import datetime
 import json
 
-from django.utils.timezone import make_aware, now
+import psycopg2 as pg
+import psycopg2.extras as pge
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from note.models import Note, NoteUser, NoteClub
-from note.models import Alias
 from member.models import Club, Profile
+from note.models import Alias, Note, NoteClub, NoteUser
 
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 M_DURATION = 396
-M_START = datetime.date(2019, 8, 31)
+M_START = datetime.date(2019, 8, 1)
 M_END = datetime.date(2020, 9, 30)
 
 MAP_IDBDE = {
@@ -32,6 +30,11 @@ MAP_IDBDE = {
 # some Aliases have been created in the fixtures
 ALIAS_SET = {a[0] for a in Alias.objects.all().values_list("normalized_name")}
 
+# Some people might loose some aliases due to normalization. We warn them on them.
+LOST_ALIASES = {}
+# In some rare cases, the username might be in conflict with some others. We change them and warn the users.
+CHANGED_USERNAMES = []
+
 note_user_type = ContentType.objects.get(app_label="note", model="noteuser")
 note_club_type = ContentType.objects.get(app_label="note", model="noteclub")
 
@@ -59,6 +62,19 @@ class Command(ImportCommand):
         Every Model has to be manually created, and no magic `.save()`
         function is being called.
         """
+        # Get promotion and date of creation of the account
+        cur.execute("SELECT idbde, MIN(date) AS created_at, MIN(annee) AS promo FROM adhesions"
+                    " GROUP BY idbde ORDER BY promo, created_at;")
+        MAP_IDBDE_PROMOTION = {}
+        for row in cur:
+            MAP_IDBDE_PROMOTION[row["idbde"]] = row
+
+        cur.execute("SELECT * FROM comptes WHERE idbde <= 0 ORDER BY idbde;")
+        for row in cur:
+            note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]])
+            note.balance = row["solde"]
+            note.save()
+
         cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
         pk_club = 3
         pk_user = 1
@@ -71,9 +87,10 @@ class Command(ImportCommand):
             pseudo = row["pseudo"]
             pseudo_norm = Alias.normalize(pseudo)
             self.update_line(idx, n, pseudo)
-            # clean pseudo (normalized pseudo must be unique)
+            # clean pseudo (normalized pseudo must be unique and not empty)
-            if pseudo_norm in ALIAS_SET:
+            if not pseudo_norm or pseudo_norm in ALIAS_SET:
                 pseudo = pseudo + str(row["idbde"])
+                CHANGED_USERNAMES.append((pk_note, row["idbde"], pseudo))
             else:
                 ALIAS_SET.add(pseudo_norm)
             # clean date
@@ -81,9 +98,8 @@ class Command(ImportCommand):
                 "pk": pk_note,
                 "balance": row['solde'],
                 "last_negative": None,
-                "is_active": True,
+                "is_active": not row["bloque"],
-                "display_image": "",
+                "display_image": "pic/default.png",
-                "created_at": now()
             }
             if row["last_negatif"] is not None:
                 note_dict["last_negative"] = make_aware(row["last_negatif"])
@@ -94,14 +110,25 @@ class Command(ImportCommand):
                 else:
                     passwd_nk15 = ''
 
+                # Note account should have no password and be active
+                if int(row["idbde"]) == 3508:
+                    passwd_nk15 = "ipbased$127.0.0.1"
+                    row["bloque"] = False
+
+                if row["idbde"] not in MAP_IDBDE_PROMOTION:
+                    # NK12 bug. Applying default values
+                    MAP_IDBDE_PROMOTION[row["idbde"]] = {"promo": 2014,
+                                                         "created_at": datetime.datetime(2014, 9, 1, 0, 0, 0)}
+
                 obj_dict = {
                     "pk": pk_user,
                     "username": row["pseudo"],
                     "password": passwd_nk15,
-                    "first_name": row["nom"],
+                    "first_name": row["prenom"],
-                    "last_name": row["prenom"],
+                    "last_name": row["nom"],
                     "email": row["mail"],
-                    "is_active": True,  # temporary
+                    "is_active": not row["bloque"],
+                    "date_joined": make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"]),
                 }
                 profile_dict = {
                     "pk": pk_profile,
@@ -111,7 +138,11 @@ class Command(ImportCommand):
                     "paid": row['normalien'],
                     "registration_valid": True,
                     "email_confirmed": True,
+                    "promotion": MAP_IDBDE_PROMOTION[row["idbde"]]["promo"],
+                    "report_frequency": max(row["report_period"], 0),
+                    "last_report": make_aware(row["previous_report_date"]),
                 }
+                note_dict["created_at"] = make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"])
                 note_dict["polymorphic_ctype"] = note_user_type
                 note_user_dict = {
                     "pk": pk_note,
@@ -137,6 +168,7 @@ class Command(ImportCommand):
                     "pk": pk_club,
                     "name": row["pseudo"],
                     "email": row["mail"],
+                    "parent_club_id": 1,   # All clubs depends on BDE by default
                     "membership_duration": M_DURATION,
                     "membership_start": M_START,
                     "membership_end": M_END,
@@ -145,7 +177,7 @@ class Command(ImportCommand):
                 }
                 note_club_dict = {
                     "pk": pk_note,
-                    "club_id": pk_club,
+                    "club_id": pk_club
                 }
                 alias_dict = {
                     "pk": pk_note,
@@ -153,6 +185,7 @@ class Command(ImportCommand):
                     "normalized_name": Alias.normalize(pseudo),
                     "note_id": pk_note
                 }
+                note_dict["created_at"] = make_aware(row["previous_report_date"])  # Not perfect, but good approximation
                 note_dict["polymorphic_ctype"] = note_club_type
                 bulk_mgr.add(Club(**obj_dict),
                              Note(**note_dict),
@@ -177,11 +210,13 @@ class Command(ImportCommand):
         pk_alias = Alias.objects.order_by('-id').first().id + 1
         for idx, row in enumerate(cur):
             alias_name = row["alias"]
-            alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
+            alias_name = (alias_name[:254] + '…') if len(alias_name) > 255 else alias_name
             alias_norm = Alias.normalize(alias_name)
             self.update_line(idx, n, alias_norm)
             # clean pseudo (normalized pseudo must be unique)
-            if alias_norm in ALIAS_SET:
+            if not alias_norm or alias_norm in ALIAS_SET:
+                LOST_ALIASES.setdefault(MAP_IDBDE[row["idbde"]], [])
+                LOST_ALIASES[MAP_IDBDE[row["idbde"]]].append(alias_name)
                 continue
             else:
                 ALIAS_SET.add(alias_norm)
@@ -212,3 +247,7 @@ class Command(ImportCommand):
             filename = kwargs["save"]
             with open(filename, 'w') as fp:
                 json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
+            with open(filename + ".changed-usernames", 'w') as fp:
+                json.dump(CHANGED_USERNAMES, fp, sort_keys=True, indent=2)
+            with open(filename + ".removed-aliases", 'w') as fp:
+                json.dump(LOST_ALIASES, fp, sort_keys=True, indent=2)

management/commands/import_activities.py

@@ -1,20 +1,32 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import psycopg2 as pg
 import psycopg2.extras as pge
-import datetime
+from activity.models import Activity, ActivityType, Entry, Guest
-import copy
-
-from django.utils.timezone import make_aware
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from activity.models import ActivityType, Activity, Guest, Entry
 from member.models import Club
-from note.models import Note
+from note.models import Note, NoteUser
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_ACTIVITY = dict()
 
+CLUB_RELOU = [
+    0,  # BDE
+    4771,  # Kataclist
+    5162,  # Assurance BDE ?!
+    5164,  # S & L
+    625,  # Aspique
+    5154,  # Frekens
+    3944,  # DiskJok[ENS]
+    5153,  # Monopo[list]
+    2351,  # JdRM
+    2365,  # Pot Vieux
+]
+
+
 class Command(ImportCommand):
     """
     Import command for Activities Base Data (Comptes, and Aliases)
@@ -26,16 +38,19 @@ class Command(ImportCommand):
         cur.execute("SELECT * FROM activites ORDER by id")
         n = cur.rowcount
         bulk_mgr = BulkCreateManager(chunk_size=chunk)
-        activity_type_id = ActivityType.objects.get(name="Pot").pk  # Need to be fixed manually
+        pot_id = ActivityType.objects.get(name="Pot").pk
+        club_id = ActivityType.objects.get(name="Soirée de club").pk
         kfet = Club.objects.get(name="Kfet")
         pk_activity = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["titre"])
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             note = self.MAP_IDBDE[row["responsable"]]
             if note == 6244:
-                 # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
+                # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
                 note = Note.objects.get(pk=self.MAP_IDBDE[6524])
-                note = note.user_id
+                note = note.id
             organizer = Club.objects.filter(name=row["signature"])
             if organizer.exists():
                 # Try to find the club that organizes the activity.
@@ -47,18 +62,19 @@ class Command(ImportCommand):
                 "pk": pk_activity,
                 "name": row["titre"],
                 "description": row["description"],
-                "activity_type_id": activity_type_id,  # By default Pot
+                "location": row["lieu"],
-                "creater_id": note,
+                "activity_type_id": pot_id if row["liste"] else club_id,
+                "creater_id": NoteUser.objects.get(pk=note).user.id,
                 "organizer_id": organizer.pk,
                 "attendees_club_id": kfet.pk,  # Maybe fix manually
                 "date_start": make_aware(row["debut"]),
                 "date_end": make_aware(row["fin"]),
                 "valid": row["validepar"] is not None,
-                "open": row["ouvert"],  # Should always be False
+                "open": False,
             }
             # WARNING: Fields lieu, liste, listeimprimee are missing
             MAP_ACTIVITY[row["id"]] = pk_activity
-            pk_activity +=1
+            pk_activity += 1
             bulk_mgr.add(Activity(**obj_dict))
         bulk_mgr.done()
 
@@ -70,6 +86,8 @@ class Command(ImportCommand):
         n = cur.rowcount
         for idx, row in enumerate(cur):
             self.update_line(idx, n, f"{row['nom']} {row['prenom']}")
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             obj_dict = {
                 "pk": row["id"],
                 "activity_id": MAP_ACTIVITY[row["activite"]],
@@ -88,11 +106,13 @@ class Command(ImportCommand):
         n = cur.rowcount
         for idx, row in enumerate(cur):
             self.update_line(idx, n, f"{row['idbde']} {row['responsable']}")
+            if row["idbde"] in CLUB_RELOU:
+                row["idbde"] = 3508
             obj_dict = {
                 "activity_id": MAP_ACTIVITY[row["activite"]],
                 "time": make_aware(row["heure_entree"]),
                 "note_id": self.MAP_IDBDE[row["responsable"] if row['est_invite'] else row["idbde"]],
-                "guest_id": self.MAP_IDBDE[row["idbde"]] if row['est_invite'] else None,
+                "guest_id": row["idbde"] if row['est_invite'] else None,
             }
             bulk_mgr.add(Entry(**obj_dict))
         bulk_mgr.done()

management/commands/import_nk15.py

@@ -1,18 +1,37 @@
-#!/usr/env/bin python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 import subprocess
 
-from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
-class Command(BaseCommand):
+from ._import_utils import ImportCommand
+
+
+class Command(ImportCommand):
     """
     Command for importing the database of NK15.
-    Need to be run by a user with a registered role in postgres for the database nk15. 
+    Need to be run by a user with a registered role in postgres for the database nk15.
     """
 
     def handle(self, *args, **kwargs):
         subprocess.call("./apps/scripts/shell/tabularasa")
-        call_command('import_account', alias=True, chunk=1000, save = "map.json")
+
-        call_command('import_activities', chunk=100, map="map.json")
+        kwargs["alias"] = True
-        call_command('import_transaction', buttons=True, map="map.json")
+        kwargs["chunk"] = 1000
-#
+        kwargs["save"] = "map.json"
+        call_command('import_account', **kwargs)
+
+        del kwargs["alias"]
+        del kwargs["save"]
+        kwargs["chunk"] = 100
+        kwargs["map"] = "map.json"
+        call_command('import_activities', **kwargs)
+
+        kwargs["chunk"] = 10000
+        kwargs["map"] = "map.json"
+        kwargs["buttons"] = True
+        call_command('import_transaction', **kwargs)
+
+        call_command('make_su','-sS', 'Coq', 'erdnaxe', 'Krokmou', 'PAC', 'Pollion', 'TLinux', 'ÿnérant')
+        call_command('syncsql')

management/commands/import_transaction.py

@@ -1,42 +1,56 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import copy
+import datetime
 import re
+
+import pytz
 import psycopg2 as pg
 import psycopg2.extras as pge
-import pytz
+from activity.models import Entry, GuestTransaction
-import datetime
+from django.contrib.auth.models import User
-import copy
+from django.contrib.contenttypes.models import ContentType
-
-from django.utils.timezone import make_aware
 from django.db import transaction
+from django.utils.timezone import make_aware
+from member.models import Membership
+from note.models import (MembershipTransaction, Note, NoteClub,
+                         RecurrentTransaction, SpecialTransaction,
+                         TemplateCategory, Transaction, TransactionTemplate)
+from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy
 
-from note.models import (TemplateCategory,
+from ._import_utils import BulkCreateManager, ImportCommand, timed
-                         TransactionTemplate,
-                         Transaction,
-                         RecurrentTransaction,
-                         SpecialTransaction
-                         )
-from note.models import Note
-from activity.models import Guest, GuestTransaction
 
-from member.models import Membership, MembershipTransaction
+MAP_TRANSACTION = dict()
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+MAP_REMITTANCE = dict()
 
+# from member/fixtures/initial
 BDE_PK = 1
 KFET_PK = 2
+
+# from note/fixtures/initial
 NOTE_SPECIAL_CODE = {
     "espèce": 1,
     "carte": 2,
     "chèque": 3,
     "virement": 4,
 }
+# from permission/fixtures/initial
+BDE_ROLE_PK = 1
+KFET_ROLE_PK = 2
 
+CT = {
+    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
+    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
+    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
+    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
+}
 
 def get_date_end(date_start):
     date_end = copy.deepcopy(date_start)
-    if date_start > 8:
+    if date_start.month >= 8:
-        date_end.year = date_start + 1
+        date_end = date_start.replace(year=date_start.year + 1)
-    date_end.month = 9
+    date_end = date_end.replace(month=9, day=30)
-    date_end.day = 30
     return date_end
 
 
@@ -48,44 +62,113 @@ class Command(ImportCommand):
     def add_arguments(self, parser):
         parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
+        parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
+                                                                                       "don't save them")
 
     @timed
-    @transaction.atomic
+    def import_buttons(self, cur, chunk_size, import_buttons):
-    def import_buttons(self, cur, chunk_size):
+        self.categories = dict()
-        categories = dict()
+        self.buttons = dict()
-        buttons = dict()
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute("SELECT * FROM boutons;")
         n = cur.rowcount
-        pk_category = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["label"])
-            if row["categorie"] not in categories:
+            if row["categorie"] not in self.categories:
-                bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"]))
+                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
-                pk_category += 1
+                cat.save()
-                categories[row["categorie"]] = pk_category
+                self.categories[row["categorie"]] = cat.pk
             obj_dict = {
                 "pk": row["id"],
                 "name": row["label"],
                 "amount": row["montant"],
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
-                "category_id": categories[row["categorie"]],
+                "category_id": self.categories[row["categorie"]],
                 "display": row["affiche"],
                 "description": row["description"],
             }
-            if row["label"] in buttons:
+            if row["label"] in self.buttons:
-                obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}"
+                obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
-            bulk_mgr.add(TransactionTemplate(**obj_dict))
+            if import_buttons:
-            buttons[obj_dict["label"]] = row["id"]
+                bulk_mgr.add(TransactionTemplate(**obj_dict))
+            self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
         bulk_mgr.done()
-        return buttons, categories
+
+    def _basic_transaction(self, row, obj_dict, child_dict):
+        if len(row["description"]) > 255:
+            obj_dict["reason"] = obj_dict["reason"][:252] + "…)"
+        return obj_dict, None, None
+
+    def _template_transaction(self, row, obj_dict, child_dict):
+        if self.buttons.get(row["description"]):
+            child_dict["template_id"] = self.buttons[row["description"]][0]
+        # elif self.categories.get(row["categorie"]):
+        #     child_dict["category_id"] = self.categories[row["categorie"]]
+        elif "WEI" in row["description"]:
+            return obj_dict, None, None
+        else:
+            return obj_dict, None, None
+        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
+        return obj_dict, child_dict, RecurrentTransaction
+
+    def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
+        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
+        obj_dict2 = obj_dict.copy()
+        child_dict2 = child_dict.copy()
+        child_dict2["membership_id"] = pk_membership
+
+        return obj_dict2, child_dict2, MembershipTransaction
+
+    def _special_transaction(self, row, obj_dict, child_dict):
+        # Some transaction uses BDE (idbde=0) as source or destination,
+        # lets fix that.
+        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
+        field_id = "source_id" if row["type"] == "crédit" else "destination_id"
+        if "espèce" in row["description"]:
+            obj_dict[field_id] = 1
+        elif "carte" in row["description"]:
+            obj_dict[field_id] = 2
+        elif "cheques" in row["description"]:
+            obj_dict[field_id] = 3
+        elif "virement" in row["description"]:
+            obj_dict[field_id] = 4
+        # humans and clubs have always the biggest id
+        actor_pk = max(row["destinataire"], row["emetteur"])
+        actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
+        # custom fields of SpecialTransaction
+        if actor.__class__.__name__ == "NoteUser":
+            child_dict["first_name"] = actor.user.first_name
+            child_dict["last_name"] = actor.user.last_name
+        else:
+            child_dict["first_name"] = actor.club.name
+            child_dict["last_name"] = actor.club.name
+        return obj_dict, child_dict, SpecialTransaction
+
+    def _guest_transaction(self, row, obj_dict, child_dict):
+        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
+        m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
+        if m:
+            activity_name = m.group(1)
+            first_name, last_name = m.group(2), m.group(3)
+            if first_name == "Marion" and last_name == "Bizu Pose":
+                first_name, last_name = "Marion Bizu", "Pose"
+            entry_id = Entry.objects.filter(
+                    activity__name__iexact=activity_name,
+                    guest__first_name__iexact=first_name,
+                    guest__last_name__iexact=last_name,
+            ).first().pk
+            child_dict["entry_id"] = entry_id
+        else:
+            raise Exception(f"Guest not Found {row['id']} first_name, last_name")
+
+        return obj_dict, child_dict, GuestTransaction
 
     @timed
     @transaction.atomic
-    def import_transaction(self, cur, chunk_size, idmin, buttons, categories):
+    def import_transaction(self, cur, chunk_size, idmin, save=True):
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute(
-            f"SELECT t.date AS transac_date, t.type, t.emetteur,\
+            f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
               t.destinataire,t.quantite, t.montant, t.description,\
               t.valide, t.cantinvalidate, t.categorie, \
               a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
@@ -96,8 +179,21 @@ class Command(ImportCommand):
         n = cur.rowcount
         pk_membership = 1
         pk_transaction = 1
+        kfet_balance = 0
         for idx, row in enumerate(cur):
-            self.update_line(idx, n, row["description"])
+            if save or idx % chunk_size == 0:
+                self.update_line(idx, n, row["description"])
+
+            MAP_TRANSACTION[row["id"]] = pk_transaction
+
+            if not save:
+                pk_transaction += 1
+                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
+                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
+                    if not isinstance(note, NoteClub):
+                        pk_transaction += 1
+                continue
+
             try:
                 date = make_aware(row["transac_date"])
             except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
@@ -107,99 +203,195 @@ class Command(ImportCommand):
             obj_dict = {
                 "pk": pk_transaction,
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
+                "polymorphic_ctype": None,
                 "source_id": self.MAP_IDBDE[row["emetteur"]],
-                "created_at": date,
                 "amount": row["montant"],
+                "created_at": date,
+                "destination_alias": "",
+                "invalidity_reason": "",
                 "quantity": row["quantite"],
                 "reason": row["description"],
+                "source_alias": "",
                 "valid": row["valide"],
             }
+            if len(obj_dict["reason"]) > 255:
+                obj_dict["reason"] = obj_dict["reason"][:254] + "…"
             # for child transaction Models
-            child_dict = {"pk": obj_dict["pk"]}
+            child_dict = {"pk": pk_transaction}
             ttype = row["type"]
-            if ttype == "don" or ttype == "transfert":
+            # Membership transaction detection and import
-                child_transaction = None
+            if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
+                note = Note.objects.get(pk=obj_dict["source_id"])
+                if isinstance(note, NoteClub):
+                    child_transaction = None    # don't bother register clubs
+                else:
+                    user_id = note.user_id
+                    montant = obj_dict["amount"]
+                    (obj_dict0,
+                     child_dict0,
+                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
+                    obj_dict0["destination_id"] = 6  # Kfet note id
+                    bde_dict = {
+                        "pk": pk_membership,
+                        "user_id": user_id,
+                        "club_id": BDE_PK,
+                        "date_start": date.date(),  # Only date, not time
+                        "date_end": get_date_end(date.date()),
+                        "fee": min(500, montant)
+                    }
+                    pk_membership += 1
+                    pk_transaction += 1
+                    obj_dict, child_dict, child_transaction =\
+                        self._membership_transaction(row, obj_dict, child_dict, pk_membership)
+                    # Kfet membership
+                    # BDE Membership
+                    obj_dict["pk"] = pk_transaction
+                    child_dict["pk"] = pk_transaction
+                    kfet_dict = {
+                        "pk": pk_membership,
+                        "user_id": user_id,
+                        "club_id": KFET_PK,
+                        "date_start": date.date(),  # Only date, not time
+                        "date_end": get_date_end(date.date()),
+                        "fee": max(montant - 500, 0),
+                    }
+                    obj_dict0["amount"] = bde_dict["fee"]
+                    obj_dict["amount"] = kfet_dict["fee"]
+                    kfet_balance += kfet_dict["fee"]
+                    # BDE membership Transaction is inserted before the Kfet membershipTransaction
+                    pk_membership += 1
+                    pk_transaction += 1
+                    bulk_mgr.add(
+                        Membership(**bde_dict),
+                        Membership(**kfet_dict),
+                        Transaction(**obj_dict0),
+                        child_transaction(**child_dict0),
+                        Transaction(**obj_dict),
+                        child_transaction(**child_dict),
+                    )
+                    continue
             elif ttype == "bouton":
-                child_transaction = RecurrentTransaction
+                obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
-                child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"])
-                child_dict["template_id"] = buttons[row["description"]]
             elif ttype == "crédit" or ttype == "retrait":
-                child_transaction = SpecialTransaction
+                obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
-                # Some transaction uses BDE (idbde=0) as source or destination,
-                # lets fix that.
-                field_id = "source_id" if ttype == "crédit" else "destination_id"
-                if "espèce" in row["description"]:
-                    obj_dict[field_id] = 1
-                elif "carte" in row["description"]:
-                    obj_dict[field_id] = 2
-                elif "cheques" in row["description"]:
-                    obj_dict[field_id] = 3
-                elif "virement" in row["description"]:
-                    obj_dict[field_id] = 4
-                # humans and clubs have always the biggest id
-                actor_pk = max(row["destinataire"], row["emetteur"])
-                actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
-                # custom fields of SpecialTransaction
-                if actor.__class__.__name__ == "NoteUser":
-                    child_dict["first_name"] = actor.user.first_name
-                    child_dict["last_name"] = actor.user.last_name
-                else:
-                    child_dict["first_name"] = actor.club.name
-                    child_dict["last_name"] = actor.club.name
-            elif ttype == "adhésion" and row["valide"]:
-                child_transaction = MembershipTransaction
-                # Kfet membership
-                montant = row["montant"]
-                obj_dict["amount"] = min(500, montant)
-                child_dict["membership_id"] = pk_membership
-                kfet_dict = {
-                    "pk": pk_membership,
-                    "user": self.MAP_IDBDE[row["idbde"]],
-                    "club": KFET_PK,
-                    "date_start": row["date"].date(),  # Only date, not time
-                    "date_end": get_date_end(row["date"].date()),
-                    "fee": min(500, montant)
-                }
-
-                pk_membership += 1
-                pk_transaction += 1
-                # BDE Membership
-                obj_dict2 = obj_dict.copy()
-                child_dict2 = dict()
-                obj_dict2["pk"] = pk_transaction
-                obj_dict2["amount"] = max(montant - 500, 0)
-                child_dict2["pk"] = pk_transaction
-                bde_dict = {
-                    "pk": pk_membership,
-                    "user": self.MAP_IDBDE[row["idbde"]],
-                    "club": BDE_PK,
-                    "date_start": row["date"].date(),  # Only date, not time
-                    "date_end": get_date_end(row["date"].date()),
-                    "fee": max(montant - 500, 0),
-                }
-                pk_membership += 1
-                # BDE membership Transaction is inserted before the Kfet membershipTransaction
-                bulk_mgr.add(
-                    Transaction(**obj_dict2),
-                    child_transaction(**child_dict2),
-                    Membership(**bde_dict),
-                    Membership(**kfet_dict),
-                )
             elif ttype == "invitation":
-                child_transaction = GuestTransaction
+                obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
-                m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
+            elif ttype == "don" or ttype == "transfert":
-                if m:
+                obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
-                    first_name, last_name = m.groups(1), m.groups(2)
+            else:
-                    guest_id = Guest.object.filter(first_name__iexact=first_name,
+                child_transaction = None
-                                                   last_name__iexact=last_name).first().pk
+                # create base transaction object and typed one
-                    child_dict["guest_id"] = guest_id
+            bulk_mgr.add(Transaction(**obj_dict))
-                else:
+            if child_transaction is not None:
-                    raise(f"Guest not Found {row['id']} {first_name}, last_name" )
+                child_dict.update(obj_dict)
-
+                bulk_mgr.add(child_transaction(**child_dict))
-            bulk_mgr.add(Transaction(**obj_dict),
-                         child_transaction(**child_dict))
             pk_transaction += 1
-           
+        bulk_mgr.done()
+
+        # Update the balance of the BDE and the Kfet club
+        note_bde = NoteClub.objects.get(pk=5)
+        note_kfet = NoteClub.objects.get(pk=6)
+        note_bde.balance -= kfet_balance
+        note_kfet.balance += kfet_balance
+        note_bde.save()
+        note_kfet.save()
+
+    @timed
+    def set_roles(self):
+        bulk_mgr = BulkCreateManager(chunk_size=10000)
+        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
+        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
+        n = len(bde_membership_ids)
+        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
+            self.update_line(idx, n, str(idx))
+            bulk_mgr.add(
+                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
+                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
+            )
+            bulk_mgr.done()
+
+        # Note account has a different treatment
+        for m in Membership.objects.filter(user_username="note").all():
+            m.date_end = "3142-12-12"
+            m.roles.set([20])   # PC Kfet role
+            m.save()
+
+    @timed
+    @transaction.atomic
+    def import_remittances(self, cur, chunk_size):
+        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
+        cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
+        n = cur.rowcount
+        pk_remittance = 1
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["commentaire"])
+
+            MAP_REMITTANCE[row["id"]] = pk_remittance
+
+            remittance_dict = {
+                "pk": pk_remittance,
+                "date": make_aware(row["date"]),
+                "remittance_type_id": 1,    # Only Bank checks are supported in NK15
+                "comment": row["commentaire"],
+                "closed": row["close"],
+            }
+            bulk_mgr.add(Remittance(**remittance_dict))
+
+            pk_remittance += 1
+
+        bulk_mgr.done()
+
+    @timed
+    def import_checks(self, cur):
+        cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
+                    "FROM cheques ORDER BY id;")
+        n = cur.rowcount
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["nom"])
+
+            if not row["idremise"]:
+                continue
+
+            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
+            tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
+            tr.save()
+
+            tr = tr.transaction
+            tr.last_name = row["nom"]
+            tr.first_name = row["prenom"]
+            tr.bank = row["banque"]
+            try:
+                tr.save()
+            except:
+                print("Failed to save row: " + str(row))
+
+    @timed
+    def import_soge_credits(self):
+        users = User.objects.filter(profile__registration_valid=True).order_by('pk')
+        n = users.count()
+        for idx, user in enumerate(users.all()):
+            self.update_line(idx, n, user.username)
+            soge_credit_transaction = SpecialTransaction.objects.filter(
+                reason__icontains="crédit sogé",
+                destination_id=user.note.id,
+            )
+            if soge_credit_transaction.exists():
+                soge_credit_transaction = soge_credit_transaction.get()
+                soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
+                memberships = Membership.objects.filter(
+                    user=user,
+                    club_id__in=[BDE_PK, KFET_PK],
+                    date_start__lte=soge_credit_transaction.created_at,
+                    date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
+                ).all()
+                for membership in memberships:
+                    soge_credit.transactions.add(membership.transaction)
+                soge_credit.save()
+
+
+    @timed
     def handle(self, *args, **kwargs):
         # default args, provided by ImportCommand.
         nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
@@ -208,8 +400,11 @@ class Command(ImportCommand):
         cur = conn.cursor(cursor_factory=pge.DictCursor)
 
         if kwargs["map"]:
-            self.load(kwargs["map"])
+            self.load_map(kwargs["map"])
-           
+        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
-        self.import_buttons(cur, kwargs["chunk"])
+        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
-       
+        if not kwargs["nosave"]:
-        self.import_transaction(cur, kwargs["chunk"])
+            self.set_roles()
+        self.import_remittances(cur, kwargs["chunk"])
+        self.import_checks(cur)
+        self.import_soge_credits()

management/commands/make_su.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.core.management.base import BaseCommand
@@ -16,7 +16,11 @@ class Command(BaseCommand):
             user = User.objects.get(username=uname)
             user.is_active = True
             if kwargs['STAFF']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to staff users…")
                 user.is_staff = True
             if kwargs['SUPER']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to superusers…")
                 user.is_superuser = True
             user.save()

management/commands/refresh_activities.py

@@ -0,0 +1,180 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from bs4 import BeautifulSoup
+from activity.models import Activity
+from django.core.management import BaseCommand
+from django.urls import reverse
+from django.utils import timezone
+
+
+class Command(BaseCommand):
+    acl_header = "#acl NoteKfet2020:read,write,admin All:read Default\n"
+
+    warning_header = """## NE PAS ÉDITER CETTE PAGE MANUELLEMENT
+## ELLE EST GÉNÉRÉE AUTOMATIQUEMENT PAR LA NOTE KFET 2020
+## Adapté par [[WikiYnerant|ÿnérant]] du script de by 20-100, largement inspiré de la version de Barbichu.
+"""
+
+    intro_generic = """ * Elle est générée automatiquement par la [[NoteKfet/NoteKfet2020|Note Kfet 2020]]
+ * Ne pas éditer cette page manuellement, toute modification sera annulée automatiquement.
+ * Pour annoncer un nouvel événement, rendez-vous sur {activities_url}
+
+""".format(activities_url="https://" + os.getenv("NOTE_URL", "") + reverse("activity:activity_list"))
+
+    @staticmethod
+    def connection(url):
+        """Se logue sur le wiki et renvoie le cookie de session"""
+        parameters = {
+            'action': 'login',
+            'login': 'Connexion',
+            'name': os.getenv("WIKI_USER", "NoteKfet2020"),
+            'password': os.getenv("WIKI_PASSWORD"),
+        }
+        # Il faut encoder ça proprement
+        data = urlencode(parameters).encode("utf-8")
+        request = Request(url, data)
+        # La requête est envoyée en HTTP POST
+        response = urlopen(request)
+        # a priori la page elle-même je m'en carre…
+        response.read(2)
+        # …ce qui m'intéresse, c'est le cookie qu'elle me file
+        cookie = response.headers['set-cookie']
+        return cookie
+
+    @staticmethod
+    def get_edition_ticket(url, cookie):
+        """Récupère le ticket d'édition de la page"""
+        # On crée la requête d'édition…
+        suffix = "?action=edit&editor=text"
+        request = Request(url + suffix)
+        # …avec le cookie
+        request.add_header("Cookie", cookie)
+        # On l'envoie
+        pagecontent = urlopen(request)
+        html = pagecontent.read()
+        soup = BeautifulSoup(html, features="lxml")
+        # On va chercher le formulaire
+        form = soup.find(name="form", attrs={"id": "editor"})
+        # On récupère le ticket dedans
+        ticket = soup.find(name="input", attrs={"name": "ticket"})
+        return ticket["value"]
+
+    @staticmethod
+    def edit_wiki(page, content, comment=''):
+        """Modifie une page du wiki"""
+        url = "https://wiki.crans.org/" + page
+
+        # On se connecte et on récupère le cookie de session
+        cookie = Command.connection(url)
+        # On demande l'édition et on récupère le ticket d'édition de la page
+        ticket = Command.get_edition_ticket(url, cookie)
+        # On construit la requête
+        data = {
+            'button_save': 'Enregistrer les modifications',
+            'category': '',
+            'comment': comment.encode("utf-8"),
+            'savetext': content.encode("utf-8"),
+            'action': 'edit',
+            'ticket': ticket
+        }
+        request = Request(url, urlencode(data).encode("utf-8"))
+        request.add_header("Cookie", cookie)
+        # On la poste
+        urlopen(request)
+
+    @staticmethod
+    def format_activity(act, raw=True):
+        """Wiki-formate une activité, pour le calendrier raw si ``raw``, pour le human-readable sinon."""
+        if raw:
+            return """== {title} ==
+ start:: {start}
+ end:: {end}
+ description:: {description} -- {club}
+ location:: {location}
+""".format(
+                title=act.name,
+                start=timezone.localtime(act.date_start).strftime("%Y-%m-%d %H:%M"),
+                end=timezone.localtime(act.date_end).strftime("%Y-%m-%d %H:%M"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
+                club=act.organizer.name,
+                location=act.location,
+            )
+        else:
+            return "|| {start} || {title} || {description} || {club} || {location} ||".format(
+                title=act.name,
+                start=timezone.localtime(act.date_start).strftime("%d/%m/%Y"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
+                club=act.organizer.name,
+                location=act.location,
+            )
+
+    @staticmethod
+    def get_raw_page():
+        page = "VieBde/PlanningSoirees/LeCalendrier"
+        header = Command.acl_header + Command.warning_header
+        header += """= Introduction =
+
+ * Cette page a pour but de recenser les activités BDE afin d'être signalées sur le calendrier de la
+ [[PageAccueil|page d'accueil]] du wiki.
+"""
+        header += Command.intro_generic
+        body = "\n".join(Command.format_activity(activity) for activity in Activity.objects.filter(valid=True)
+                         .order_by('-date_start').all())
+        footer = "\n----\nCatégorieCalendrierCampus"
+        return page, header + body + footer
+
+    @staticmethod
+    def get_human_readable_page():
+        page = "VieBde/PlanningSoirees"
+        header = Command.acl_header + Command.warning_header
+        header += """= Planning de soirées =
+== Introduction ==
+ * Cette page est destinée à accueillir le planning des soirées BDE.
+"""
+        header += Command.intro_generic + "\n"
+        body = """== Planning des activités à venir ==
+||'''Date'''||'''Titre'''||'''Description'''||'''Par''' ||'''Lieu'''||
+"""
+        body += "\n".join(Command.format_activity(activity, False) for activity in Activity.objects
+                          .filter(valid=True, date_end__gte=timezone.now()).order_by('-date_start').all())
+        body += """\n\n== Planning des activités passées ==
+||'''Date'''||'''Titre'''||'''Description'''||'''Par'''||'''Lieu'''||
+"""
+        body += "\n".join(Command.format_activity(activity, False) for activity in Activity.objects
+                          .filter(valid=True, date_end__lt=timezone.now()).order_by('-date_start').all())
+        return page, header + body
+
+    @staticmethod
+    def refresh_raw_wiki_page(comment="refresh", print_stdout=False, edit_wiki=False):
+        page, content = Command.get_raw_page()
+        if print_stdout:
+            print(content)
+        if edit_wiki:
+            Command.edit_wiki(page, content, comment)
+
+    @staticmethod
+    def refresh_human_readable_wiki_page(comment="refresh", print_stdout=False, edit_wiki=False):
+        page, content = Command.get_human_readable_page()
+        if print_stdout:
+            print(content)
+        if edit_wiki:
+            Command.edit_wiki(page, content, comment)
+
+    def add_arguments(self, parser):
+        parser.add_argument("--human", "-H", action="store_true", help="Save human readable page")
+        parser.add_argument("--raw", "-r", action="store_true", help="Save raw page, for the calendar")
+        parser.add_argument("--comment", "-c", action="store", type=str, default="", help="Comment of the modification")
+        parser.add_argument("--stdout", "-o", action="store_true", help="Render the wiki page in stdout")
+        parser.add_argument("--wiki", "-w", action="store_true", help="Send modifications to the wiki")
+
+    def handle(self, *args, **options):
+        if options["raw"]:
+            Command.refresh_raw_wiki_page(options["comment"], options["stdout"], options["wiki"])
+        if options["human"]:
+            Command.refresh_human_readable_wiki_page(options["comment"], options["stdout"], options["wiki"])
+

management/commands/refresh_highlighted_buttons.py

@@ -0,0 +1,29 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from django.utils import timezone
+from note.models import RecurrentTransaction, TransactionTemplate
+
+
+class Command(BaseCommand):
+    """
+    Command to add the ten most used buttons of the past month to the highlighted buttons.
+    """
+    def handle(self, *args, **kwargs):
+        queryset = RecurrentTransaction.objects.filter(
+            template__display=True,
+            valid=True,
+            created_at__gte=timezone.now() - timedelta(days=30),
+        ).values("template").annotate(transaction_count=Count("template")).order_by("-transaction_count")[:10]
+        for d in queryset.all():
+            button_id = d["template"]
+            button = TransactionTemplate.objects.get(pk=button_id)
+            if kwargs['verbosity'] > 0:
+                self.stdout.write(self.style.WARNING("Highlight button {name} ({count:d} transactions)..."
+                                                     .format(name=button.name, count=d["transaction_count"])))
+            button.highlighted = True
+            button.save()

management/commands/send_mail_to_negative_balances.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date, timedelta
+
+from django.core.mail import send_mail
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils.translation import activate
+from note.models import Note
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--spam", "-s", action='store_true', help="Spam negative users")
+        parser.add_argument("--report", "-r", action='store_true', help="Report the list of negative users to admins")
+        parser.add_argument("--negative-amount", "-n", action='store', type=int, default=1000,
+                            help="Maximum amount to be considered as very negative (inclusive)")
+        parser.add_argument("--add-years", "-y", action='store', type=int, default=0,
+                            help="Add also people that have a negative balance since N years "
+                                 "(default to only active members)")
+
+    def handle(self, *args, **options):
+        activate('fr')
+
+        if options['negative_amount'] == 0:
+            # Don't log empty notes
+            options['negative_amount'] = 1
+
+        notes = Note.objects.filter(
+            Q(noteuser__user__memberships__date_end__gte=
+              date.today() - timedelta(days=int(365.25 * options['add_years'])))
+            | (Q(noteclub__isnull=False) & ~Q(noteclub__club__name__icontains='- BDE')),
+            balance__lte=-options["negative_amount"],
+            is_active=True, 
+        ).order_by('balance').distinct().all()
+
+        if options["spam"]:
+            for note in notes:
+                note.send_mail_negative_balance()
+
+        if options["report"]:
+            plain_text = render_to_string("note/mails/negative_notes_report.txt", context=dict(notes=notes))
+            html = render_to_string("note/mails/negative_notes_report.html", context=dict(notes=notes))
+            send_mail("[Note Kfet] Liste des négatifs", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                      recipient_list=["respo-info.bde@lists.crans.org", "tresorerie.bde@lists.crans.org"],
+                      html_message=html)

management/commands/send_reports.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils import timezone
+from django.utils.translation import activate
+from note.models import NoteUser, Transaction
+from note.tables import HistoryTable
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--notes', '-n', type=int, nargs='+', help='Select note ids')
+        parser.add_argument('--debug', '-d', action='store_true', help='Debug mode, print mails in stdout')
+
+    def handle(self, *args, **options):
+        activate('fr')
+        if 'notes' in options and options['notes']:
+            notes = NoteUser.objects.filter(pk__in=options['notes']).all()
+        else:
+            notes = NoteUser.objects.filter(
+                user__memberships__date_end__gte=timezone.now(),
+                user__profile__report_frequency__gt=0,
+            ).distinct().all()
+        for note in notes:
+            now = timezone.now()
+            last_report = note.user.profile.last_report
+            delta = now.date() - last_report.date()
+            if delta.days < note.user.profile.report_frequency:
+                continue
+            if not options["debug"]:
+                note.user.profile.last_report = now
+                note.user.profile.save()
+            last_transactions = Transaction.objects.filter(
+                Q(source=note) | Q(destination=note),
+                created_at__gte=last_report,
+            ).order_by("created_at").all()
+            if not last_transactions.exists():
+                continue
+
+            table = HistoryTable(last_transactions)
+            incoming = sum(tr.total for tr in last_transactions if tr.destination.pk == note.pk if tr.valid)
+            outcoming = sum(tr.total for tr in last_transactions if tr.source.pk == note.pk if tr.valid)
+            context = dict(
+                user=note.user,
+                table=table,
+                last_transactions=last_transactions,
+                incoming=incoming,
+                outcoming=outcoming,
+                diff=incoming - outcoming,
+                now=now,
+                last_report=last_report,
+            )
+            plain = render_to_string("note/mails/weekly_report.txt", context)
+            html = render_to_string("note/mails/weekly_report.html", context)
+            if options["debug"]:
+                self.stdout.write(plain)
+            else:
+                note.user.email_user("[Note Kfet] Rapport de la Note Kfet", plain, html_message=html)

management/commands/send_summary_notes_report.py

@@ -0,0 +1,144 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date
+
+from django.core.mail import send_mail
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils.translation import activate
+from note.models import NoteUser, NoteClub
+from treasury.models import NoteSummary
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--negative-amount", "-n", action='store', type=int, default=1000,
+                            help="Maximum amount to be considered as very negative (inclusive)")
+
+    def handle(self, *args, **options):
+        activate('fr')
+
+        if options['negative_amount'] == 0:
+            # Don't log empty notes
+            options['negative_amount'] = 1
+    
+        
+        # User notes
+        positive_user_notes = NoteUser.objects.filter( Q(balance__gt=0) ).distinct()
+        positive_user_notes_bde = positive_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        zero_user_notes = NoteUser.objects.filter( Q(balance=0) ).distinct()
+        zero_user_notes_bde = zero_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        negative_user_notes = NoteUser.objects.filter( Q(balance__lt=0) ).distinct()
+        negative_user_notes_bde = negative_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        vnegative_user_notes = NoteUser.objects.filter( Q(balance__lte=-options["negative_amount"]) ).distinct()
+        vnegative_user_notes_bde = vnegative_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+
+        total_positive_user = positive_user_notes.count()
+        balance_positive_user = sum(note.balance for note in positive_user_notes.all())
+
+        total_positive_user_bde = positive_user_notes_bde.count()
+        balance_positive_user_bde = sum(note.balance for note in positive_user_notes_bde.all())
+
+        total_zero_user = zero_user_notes.count()
+
+        total_zero_user_bde = zero_user_notes_bde.count()
+
+        total_negative_user = negative_user_notes.count()
+        balance_negative_user = sum(note.balance for note in negative_user_notes.all())
+
+        total_negative_user_bde = negative_user_notes_bde.count()
+        balance_negative_user_bde = sum(note.balance for note in negative_user_notes_bde.all())
+
+        total_vnegative_user = vnegative_user_notes.count()
+        balance_vnegative_user = sum(note.balance for note in vnegative_user_notes.all())
+
+        total_vnegative_user_bde = vnegative_user_notes_bde.count()
+        balance_vnegative_user_bde = sum(note.balance for note in vnegative_user_notes_bde.all())
+
+        
+
+        #Club notes
+        positive_club_notes = NoteClub.objects.filter( Q(balance__gt=0) ).distinct()
+        positive_club_notes_nbde = positive_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+        zero_club_notes = NoteClub.objects.filter( Q(balance=0) ).distinct()
+        zero_club_notes_nbde = zero_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+        negative_club_notes = NoteClub.objects.filter( Q(balance__lt=0) ).distinct()
+        negative_club_notes_nbde = negative_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+
+        total_positive_club = positive_club_notes.count()
+        balance_positive_club = sum(note.balance for note in positive_club_notes.all())
+
+        total_positive_club_nbde = positive_club_notes_nbde.count()
+        balance_positive_club_nbde = sum(note.balance for note in positive_club_notes_nbde.all())
+
+        total_zero_club = zero_club_notes.count()
+
+        total_zero_club_nbde = zero_club_notes_nbde.count()
+
+        total_negative_club = negative_club_notes.count()
+        balance_negative_club = sum(note.balance for note in negative_club_notes.all())
+
+        total_negative_club_nbde = negative_club_notes_nbde.count()
+        balance_negative_club_nbde = sum(note.balance for note in negative_club_notes_nbde.all())
+        
+
+        last_summary = NoteSummary.objects.order_by('-date').first()
+
+        summary = NoteSummary.objects.create(
+            total_positive_user=total_positive_user,
+            balance_positive_user=balance_positive_user,
+
+            total_positive_user_bde=total_positive_user_bde,
+            balance_positive_user_bde=balance_positive_user_bde,
+
+            total_zero_user=total_zero_user,
+
+            total_zero_user_bde=total_zero_user_bde,
+
+            total_negative_user=total_negative_user,
+            balance_negative_user=balance_negative_user,
+
+            total_negative_user_bde=total_negative_user_bde,
+            balance_negative_user_bde=balance_negative_user_bde,
+
+            total_vnegative_user=total_vnegative_user,
+            balance_vnegative_user=balance_vnegative_user,
+
+            total_vnegative_user_bde=total_vnegative_user_bde,
+            balance_vnegative_user_bde=balance_vnegative_user_bde,
+
+
+            total_positive_club=total_positive_club,
+            balance_positive_club=balance_positive_club,
+
+            total_positive_club_nbde=total_positive_club_nbde,
+            balance_positive_club_nbde=balance_positive_club_nbde,
+
+            total_zero_club=total_zero_club,
+
+            total_zero_club_nbde=total_zero_club_nbde,
+
+            total_negative_club=total_negative_club,
+            balance_negative_club=balance_negative_club,
+
+            total_negative_club_nbde=total_negative_club_nbde,
+            balance_negative_club_nbde=balance_negative_club_nbde,
+        )
+
+        balance_difference_user = (balance_positive_user - balance_negative_user) - (last_summary.balance_positive_user - last_summary.balance_negative_user)
+        balance_difference_club = (balance_positive_club - balance_negative_club) - (last_summary.balance_positive_club - last_summary.balance_negative_club)
+
+        plain_text = render_to_string("note/mails/summary_notes_report.txt", context=dict(summary=summary, balance_difference_user=balance_difference_user, balance_difference_club=balance_difference_club))
+        html = render_to_string("note/mails/summary_notes_report.html", context=dict(summary=summary, balance_difference_user=balance_difference_user, balance_difference_club=balance_difference_club))
+        send_mail("[Note Kfet] Récapitulatif de trésorerie", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                  recipient_list=["respo-info.bde@lists.crans.org", "tresorerie.bde@lists.crans.org"],
+                  html_message=html)

management/commands/syncsql.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.apps import apps
+from django.core.management.base import BaseCommand
+from django.db import connection
+from polymorphic.models import PolymorphicModel
+
+NO_SEQ = [
+    "Session",
+    "Token",
+    "WEIRole",  # dirty fix
+]
+
+
+class Command(BaseCommand):
+    """
+    Command to synchronise primary sequence of postgres after bulk insert of django.
+    """
+
+    def add_arguments(self,parser):
+        parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
+        return parser
+
+    def handle(self, *args, **kwargs):
+        app_list = kwargs["apps"]
+        if len(app_list):
+            model_classes = list()
+            for app in app_list:
+                model_classes += apps.get_app_config(app).get_models()
+        else:
+            # no app specified, sync everything
+            model_classes = apps.get_models(include_auto_created=True)
+
+        db_names = [
+            m._meta.db_table for m in model_classes
+            if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count() > 1
+        ]
+        com = "BEGIN;\n"
+        for db_name in db_names:
+            com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1),' \
+                   f' max("id") IS NOT null) FROM "{db_name}";\n'
+        com += "COMMIT;"
+        print(com)
+        cur = connection.cursor()
+        cur.execute(com)
+        cur.close()

shell/backup_db

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Create temporary backups directory
+mkdir -p /tmp/note-backups
+date=$(date +%Y-%m-%d)
+# Backup database and save it as tar archive
+sudo -u postgres pg_dump -F t note_db > "/tmp/note-backups/$date.sql"
+# Compress backup as gzip
+gzip "/tmp/note-backups/$date.sql"
+scp "/tmp/note-backups/$date.sql.gz" "club-bde@zamok.crans.org:backup/$date.sql.gz"

shell/docker_bash

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ -r Dockerfile ]; then
+    if [ -w /var/run/docker.sock ]; then
+        docker build -t nk20 .
+        docker run -it -u $(id -u):$(id -g) --rm -v "$(pwd):/var/www/note_kfet/" -p 80:8080 nk20 bash
+    else
+        echo "Merci de rejoindre le groupe docker (ou lancez ce script en sudo) afin de pouvoir vous connecter au socket Docker."
+    fi
+else
+    echo "N'exécutez ce fichier que dans la racine de votre projet, afin de pouvoir localiser le fichier Dockerfile."
+fi

shell/tabularasa

@@ -1,6 +1,7 @@
 #!/usr/bin/sh
+sudo service postgresql stop
+sudo service postgresql start
 sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
 echo 'reset db';
-source "env/bin/activate"
 ./manage.py migrate
 ./manage.py loaddata initial

signals.py

@@ -1,38 +0,0 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import os
-import sys
-
-from django.conf import settings
-from django.core.mail import send_mail
-from django.template.loader import render_to_string
-from django.views.debug import ExceptionReporter
-
-
-def send_mail_on_exception(request, **kwargs):
-    """
-    When an error occurs on the Note Kfet, a mail is automatically sent to the webmasters.
-    """
-
-    if settings.DEBUG:
-        # Don't need to send a mail in debug mode, errors are already displayed in console and in the response view.
-        return
-
-    try:
-        exc_info = sys.exc_info()
-        exc_type = exc_info[0]
-        exc = exc_info[1]
-        tb = exc_info[2]
-        reporter = ExceptionReporter(request=request, exc_type=exc_type, exc_value=exc, tb=tb)
-
-        note_sender = os.getenv("NOTE_MAIL", "notekfet@example.com")
-        webmaster = os.getenv("WEBMASTER_MAIL", "notekfet@example.com")
-
-        message = render_to_string('scripts/mail-error500.txt', context={"error": reporter.get_traceback_text()})
-        message_html = render_to_string('scripts/mail-error500.html', context={"error": reporter.get_traceback_html()})
-
-        send_mail("Erreur dans la Note Kfet", message, note_sender, [webmaster], html_message=message_html)
-    except BaseException as e:
-        sys.stderr.write("Une erreur est survenue lors de l'envoi d'un mail, pour signaler une erreur.")
-        raise e

templates/scripts/deleted_aliases.txt

@@ -0,0 +1,27 @@
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Ce message vous est envoyé automatiquement par la Note Kfet du BDE de
+l'ENS Cachan, à laquelle vous êtes inscrit·e. Si vous n'êtes plus
+adhérent·e, vous n'êtes pas nécessairement concerné·e par la suite
+de ce message.
+
+La Note Kfet 2020 vient d'être déployée, succédant à la Note Kfet 2015.
+Les données ont été migrées.
+
+Toutefois, la nouvelle note utilise un algorithme de normalisation des alias
+permettant de rechercher plus facilement un nom de note, et empêchant la
+création d'un alias trop proche d'un autre.
+
+Nous vous informons que les alias suivants ont été supprimés de votre compte,
+jugés trop proches d'autres alias déjà existants :
+
+{{ aliases_list|join:", " }}
+
+Nous nous excusons pour le désagrément, et espérons que vous pourrez
+profiter de la nouvelle Note Kfet.
+
+Cordialement,
+
+--
+Le BDE
+

templates/scripts/unsupported_username.txt

@@ -0,0 +1,27 @@
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Ce message vous est envoyé automatiquement par la Note Kfet du BDE de
+l'ENS Cachan, à laquelle vous êtes inscrit·e. Si vous n'êtes plus
+adhérent·e, vous n'êtes pas nécessairement concerné·e par la suite
+de ce message.
+
+La Note Kfet 2020 vient d'être déployée, succédant à la Note Kfet 2015.
+Les données ont été migrées.
+
+Toutefois, la nouvelle note utilise un algorithme de normalisation des alias
+permettant de rechercher plus facilement un nom de note, et empêchant la
+création d'un alias trop proche d'un autre.
+
+Nous vous informons que votre pseudo {{ old_username }} fait pas partie des
+alias problématiques. Il a été remplacé par le pseudo {{ new_username }},
+que vous devrez utiliser pour pouvoir vous connecter. Il sera ensuite
+possible de modifier votre pseudo.
+
+Nous nous excusons pour le désagrément, et espérons que vous pourrez
+profiter de la nouvelle Note Kfet.
+
+Cordialement,
+
+--
+Le BDE
+