Merge branch 'merge_club' into 'master'

add script for notify club about their food

See merge request bde/nk20-scripts!12

README.md

@@ -1,63 +1,3 @@
 # Script de la NoteKfet 2020
 
-## Commandes Django
+La documentation est disponible sur <https://note.crans.org/doc/scripts/>.
-
-> les commandes sont documentées:
-> `./manage.py command --help`
-
-- `import_nk15` :
-
-    Importe un dump de la NoteKfet 2015. 
-
-- `make_su [--STAFF|-s] [--SUPER|-S]` :
-
-    Rend actifs les pseudos renseignés.
-  
-    * Si `--STAFF` ou `-s` est renseigné, donne en plus le statut d'équipe aux pseudos renseignés, 
-    permettant l'accès à l'interface admin.
-    * Si `--SUPER` ou `-S` est renseigné, donne en plus le statut de super-utilisateur aux pseudos renseignés,
-    octroyant tous les droits sur la plateforme.
-
-- `wei_algorithm` :
-
-    Lance l'algorithme de répartition des 1A au dernier WEI. Cela a pour effet de suggérer un bus pour tous les 1A
-    inscrits au dernier WEI en fonction des données rentrées dans le sondage, la validation se faisant ensuite
-    manuellement via l'interface Web.
-
-- `extract_ml_registrations --type {members, clubs, events, art, sport} [--year|-y YEAR]` :
-
-    Récupère la liste des adresses mail à inscrire à une liste de diffusion donnée.
-    
-    * `members` : Liste des adresses mail des utilisateurs ayant une adhésion BDE (et non Kfet) active.
-    * `clubs` : Liste des adresses mail de contact de tous les clubs BDE enregistrés.
-    * `events` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                 la liste de diffusion des événements du BDE.
-    * `art` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-              la liste de diffusion concertnant les actualités artistiques du BDA.
-    * `sport` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                la liste de diffusion concertnant les actualités sportives du BDS.
-    
-    Le champ `--year` est optionnel : il permet de choisir l'année du WEI en question (pour les trois dernières 
-    options). Si non renseigné, il s'agit du dernier WEI.
-    
-    Par défaut, si `--type` est non renseigné, la liste des adhérents BDE est renvoyée.
-
-- `extract_wei_registrations [--year|-y YEAR] [--bus|-b BUS] [--team|-t TEAM] [--sep SEP]` :
-
-    Récupère la liste des inscriptions au WEI et l'exporte au format CSV. Arguments possibles, optionnels :
-    
-    * `--year YEAR` : sélectionne l'année du WEI. Par défaut, il s'agit du dernier WEI ayant eu lieu.
-    * `--bus BUS` : filtre par bus, en récupérant uniquement les inscriptions sur un bus. Par défaut, on affiche
-    tous les bus.
-    * `--team TEAM` : filtre par équipe, en récupérant uniquement les inscriptions sur une équipe. Par défaut, on
-    affiche toutes les équipes. Entrer `"none"` filtre les inscriptions sans équipe (chefs de bus, ...)
-    * `--sep` : définit le caractère de séparation des colonnes du fichier CSV. Par défaut, il s'agit du caractère `|`.
-    Merci de ne pas rentrer plus d'un caractère.
-    
-
-## Shell
-
-- Tabula rasa :
-```shell script
-sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"
-```

__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 default_app_config = 'scripts.apps.ScriptsConfig'

apps.py

@@ -1,8 +1,7 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.apps import AppConfig
-from django.core.signals import got_request_exception
 
 
 class ScriptsConfig(AppConfig):

management/commands/_import_utils.py

@@ -1,13 +1,13 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import json
 import time
 from collections import defaultdict
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import transaction
-
 from polymorphic.models import PolymorphicModel
 
 
@@ -15,6 +15,7 @@ def timed(method):
     """"
     A simple decorator to measure time elapsed in class function (hence the args[0])
     """
+
     def _timed(*args, **kw):
         ts = time.time()
         result = method(*args, **kw)
@@ -44,14 +45,14 @@ class ImportCommand(BaseCommand):
         n = str(n)
         total = str(total)
         n.rjust(len(total))
-        print(f"\r ({n}/{total}) {content:10.10}", end="")
+        print(f"\r ({n}/{total}) {content:16.16}", end="")
 
     def create_parser(self, prog_name, subcommand, **kwargs):
         parser = super().create_parser(prog_name, subcommand, **kwargs)
         parser.add_argument('--nk15db', action='store', default='nk15', help='NK15 database name')
         parser.add_argument('--nk15user', action='store', default='nk15_user', help='NK15 database owner')
-        parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
+        parser.add_argument('-s', '--save', default='map.json', action='store', help="save mapping of idbde")
-        parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
+        parser.add_argument('-m', '--map', default='map.json', action='store', help="import mapping of idbde")
         parser.add_argument('-c', '--chunk', type=int, default=100, help="chunk size for bulk_create")
         return parser
 

management/commands/anonymize_data.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2018-2023 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+
+class Command(BaseCommand):
+    """
+    Command to protect sensitive data during the beta phase or after WEI.
+    Phone number, email address, postal address, first and last name,
+    IP addresses, health issues, gender and birth date are removed.
+    """
+    def add_arguments(self, parser):
+        parser.add_argument('--force', '-f', action='store_true', help="Actually anonymize data.")
+        parser.add_argument('--type', '-t', choices=["all", "wei", "user"], default="",
+                            help='Select the type of data to anonyze (default None)')
+
+    def handle(self, *args, **options):
+        if not options['force']:
+            if options['type'] == "all":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset ALL personal data with "
+                              "sample data. Don't use  in production! If you know what you are doing, please "
+                              "add --force option.")
+            elif options['type'] == "wei":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset WEI personal data with "
+                              "sample data. Use it in production only after WEI. If you know what you are doing,"
+                              "please add --force option.")
+            elif options['type'] == "user":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data "
+                              "visible by user (not admin or trez BDE) with sample data. Don't use  in "
+                              "production! If you know what you are doing, please add --force option.")
+            else:
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data with "
+                              "sample data. Don't use  in production ('wei' can be use in production after "
+                              "the WEI)! If you know what you are doing, please choose a type.")
+            exit(1)
+
+        cur = connection.cursor()
+        if options['type'] in ("all","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize profile, user club and guest data")
+            cur.execute("UPDATE member_profile SET "
+                    "phone_number = '0123456789', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE auth_user SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme', "
+                    "email = 'anonymous@example.com';")
+            cur.execute("UPDATE member_club SET "
+                    "email = 'anonymous@example.com';")
+            cur.execute("UPDATE activity_guest SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+
+        if options['type'] in ("all","wei","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize WEI data")
+            cur.execute("UPDATE wei_weiregistration SET "
+                    "birth_date = '1998-01-08', "
+                    "emergency_contact_name = 'Anne Onyme', "
+                    "emergency_contact_phone = '0123456789', "
+                    "gender = 'nonbinary', "
+                    "health_issues = 'Tout va bien';")
+
+        if options['type'] == "all":
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize invoice, special transaction, log, mailer and oauth data")
+            cur.execute("UPDATE treasury_invoice SET "
+                    "name = 'Anne Onyme', "
+                    "object = 'Rends nous riches', "
+                    "description = 'Donne nous plein de sous', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE treasury_product SET "
+                    "designation = 'un truc inutile';")
+            cur.execute("UPDATE note_specialtransaction SET "
+                    "bank = 'le matelas', "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+            cur.execute("UPDATE logs_changelog SET "
+                    "ip = '127.0.0.1', "
+                    "data = 'new data', "
+                    "previous = 'old data';")
+            cur.execute("UPDATE mailer_messagelog SET "
+                    "log_message = 'log message', "
+                    "message_data = 'message data';")
+            cur.execute("UPDATE mailer_dontsendentry SET "
+                    "to_address = 'anonymous@example.com';")
+            cur.execute("UPDATE oauth2_provider_application SET "
+                    "name = 'external app', "
+                    "redirect_uris = 'http://external.app', "
+                    "client_secret = 'abcdefghijklmnopqrstuvwxyz';")
+        cur.close()

management/commands/check_consistency.py

@@ -0,0 +1,54 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management import BaseCommand
+from django.db.models import Sum, F
+
+from note.models import Note, Transaction
+from note.templatetags.pretty_money import pretty_money
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--sum-all', '-s', action='store_true', help='Check if the global sum is equal to zero')
+        parser.add_argument('--check-all', '-a', action='store_true', help='Check all notes')
+        parser.add_argument('--check', '-c', type=int, nargs='+', help='Select note ids')
+        parser.add_argument('--fix', '-f', action='store_true', help='Fix note balances')
+
+    def handle(self, *args, **options):
+        error = False
+        err_log = ""
+
+        if options["sum_all"]:
+            s = Note.objects.aggregate(Sum("balance"))["balance__sum"]
+            if s:
+                self.stderr.write(self.style.NOTICE("LA SOMME DES NOTES NE VAUT PAS ZÉRO : " + pretty_money(s)))
+                error = True
+            else:
+                if options["verbosity"] > 0:
+                    self.stdout.write(self.style.SUCCESS("La somme des notes vaut bien zéro."))
+
+        notes = Note.objects.none()
+        if options["check_all"]:
+            notes = Note.objects.all()
+        elif options["check"]:
+            notes = Note.objects.filter(pk__in=options["check"])
+
+        for note in notes:
+            balance = note.balance
+            incoming = Transaction.objects.filter(valid=True, destination=note)\
+                .annotate(total=F("quantity") * F("amount")).aggregate(Sum("total"))["total__sum"] or 0
+            outcoming = Transaction.objects.filter(valid=True, source=note)\
+                .annotate(total=F("quantity") * F("amount")).aggregate(Sum("total"))["total__sum"] or 0
+            calculated_balance = incoming - outcoming
+            if calculated_balance != balance:
+                self.stderr.write(self.style.NOTICE(f"LA SOMME DES TRANSACTIONS DE LA NOTE {note} NE CORRESPOND PAS "
+                                                    "AVEC LE MONTANT RÉEL"))
+                self.stderr.write(self.style.NOTICE(f"Attendu : {pretty_money(balance)}, "
+                                                    f"calculé : {pretty_money(calculated_balance)}"))
+                if options["fix"]:
+                    note.balance = calculated_balance
+                    note.save()
+                error = True
+
+        exit(1 if error else 0)

management/commands/compilejsmessages.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import translation
+from django.views.i18n import JavaScriptCatalog
+
+
+class Command(BaseCommand):
+    """
+    Generate Javascript translation files
+    """
+
+    def handle(self, *args, **kwargs):
+        for code, _ in settings.LANGUAGES:
+            if code == settings.LANGUAGE_CODE:
+                continue
+            if kwargs["verbosity"] > 0:
+                self.stdout.write(f"Generate {code} javascript localization file")
+            with translation.override(code):
+                resp = JavaScriptCatalog().get(None, packages="member+note")
+                if not os.path.isdir(settings.STATIC_ROOT + "/js/jsi18n"):
+                    os.makedirs(settings.STATIC_ROOT + "/js/jsi18n")
+                with open(settings.STATIC_ROOT + f"/js/jsi18n/{code}.js", "wb") as f:
+                    f.write(resp.content)

management/commands/extract_ml_registrations.py

@@ -0,0 +1,136 @@
+# Copyright (C) 2018-2024 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date
+
+from django.contrib.auth.models import User
+from django.core.management import BaseCommand
+from member.models import Club, Membership
+from django.core.mail import send_mail
+
+
+class Command(BaseCommand):
+    help = "Get mailing list registrations from the last wei. " \
+           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -l {fr, en} -y {0, 1, ...}. " \
+           "You can write this into a file with a pipe, then paste the document into your mail manager."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--type', '-t', choices=["members", "clubs", "events", "art", "sport"], default="members",
+                            help='Select the type of the mailing list (default members)')
+        parser.add_argument('--lang', '-l', type=str, choices=['fr', 'en'], default='fr',
+                            help='Select the registred users of the ML of the given language. Useful only for the '
+                                 'events mailing list.')
+        parser.add_argument('--years', '-y', type=int, default=0, 
+                            help='Select the cumulative registred users of a membership from years ago. 0 means the current users')
+        parser.add_argument('--email', '-e', type=str, default="", 
+                            help='Put the email supposed to receive the emails of the mailing list (only for art). If nothing    is put, the script will just print the emails.')
+
+    def handle(self, *args, **options):
+        # TODO: Improve the mailing list extraction system, and link it automatically with Mailman.
+
+        if options['verbosity'] == 0:
+            # This is useless, but this what the user asked.
+            return
+
+        if options["type"] == "members":
+            today_date = date.today()
+            selected_date = date(today_date.year - options["years"], today_date.month, today_date.day)
+            for membership in Membership.objects.filter(
+                club__name="BDE",
+                date_start__lte=today_date,
+                date_end__gte=selected_date,
+            ).all():
+                self.stdout.write(membership.user.email)
+            return
+
+        if options["type"] == "clubs":
+            for club in Club.objects.all():
+                self.stdout.write(club.email)
+            return
+
+        # Get the list of mails that want to be registered to the events mailing listn, as well as the number of mails.
+        # Print it or send it to the email provided by the user.
+        # Don't filter to valid members, old members can receive these mails as long as they want.
+        if options["type"] == "events":
+            nb=0
+
+            if options["email"] == "":
+                for user in User.objects.filter(profile__ml_events_registration=options["lang"]).all():
+                    self.stdout.write(user.email)     
+                    nb+=1
+                self.stdout.write(str(nb))
+
+            else :
+                emails = []
+                for user in User.objects.filter(profile__ml_events_registration=options["lang"]).all():
+                    emails.append(user.email)
+                    nb+=1
+
+                subject = "Liste des abonnés à la newsletter BDE"
+                message = (
+                    f"Voici la liste des utilisateurs abonnés à la newsletter BDE:\n\n"
+                    + "\n".join(emails) 
+                    + f"\n\nTotal des abonnés : {nb}"
+                )
+                from_email = "Note Kfet 2020 <notekfet2020@crans.org>" 
+                recipient_list = [options["email"]] 
+
+                send_mail(subject, message, from_email, recipient_list)
+
+            return
+
+        if options["type"] == "art":
+            nb=0
+
+            if options["email"] == "":
+                for user in User.objects.filter(profile__ml_art_registration=True).all():
+                    self.stdout.write(user.email)     
+                    nb+=1
+                self.stdout.write(str(nb))
+
+            else :
+                emails = []
+                for user in User.objects.filter(profile__ml_art_registration=True).all():
+                    emails.append(user.email)
+                    nb+=1
+
+                subject = "Liste des abonnés à la newsletter BDA"
+                message = (
+                    f"Voici la liste des utilisateurs abonnés à la newsletter BDA:\n\n"
+                    + "\n".join(emails) 
+                    + f"\n\nTotal des abonnés : {nb}"
+                )
+                from_email = "Note Kfet 2020 <notekfet2020@crans.org>" 
+                recipient_list = [options["email"]] 
+
+                send_mail(subject, message, from_email, recipient_list)
+
+            return
+
+        if options["type"] == "sport":
+            nb=0
+
+            if options["email"] == "":
+                for user in User.objects.filter(profile__ml_sport_registration=True).all():
+                    self.stdout.write(user.email)     
+                    nb+=1
+                self.stdout.write(str(nb))
+
+            else :
+                emails = []
+                for user in User.objects.filter(profile__ml_sport_registration=True).all():
+                    emails.append(user.email)
+                    nb+=1
+
+                subject = "Liste des abonnés à la newsletter BDS"
+                message = (
+                    f"Voici la liste des utilisateurs abonnés à la newsletter BDS:\n\n"
+                    + "\n".join(emails) 
+                    + f"\n\nTotal des abonnés : {nb}"
+                )
+                from_email = "Note Kfet 2020 <notekfet2020@crans.org>" 
+                recipient_list = [options["email"]] 
+
+                send_mail(subject, message, from_email, recipient_list)
+
+            return

management/commands/force_delete_user.py

@@ -0,0 +1,176 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import getpass
+from time import sleep
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.mail import mail_admins
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.db.models import Q
+from django.test import override_settings
+from note.models import Alias, Transaction
+
+
+class Command(BaseCommand):
+    """
+    This script is used to force delete a user.
+    THIS IS ONLY ATTENDED TO BE USED TO DELETE FAKE ACCOUNTS THAT
+    WERE VALIDATED BY ERRORS. Please use data anonymization if you
+    want to block the account of a user.
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument('user', type=str, nargs='+', help="User id to delete.")
+        parser.add_argument('--force', '-f', action='store_true',
+                            help="Force the script to have low verbosity.")
+        parser.add_argument('--doit', '-d', action='store_true',
+                            help="Don't ask for a final confirmation and commit modification. "
+                                 "This option should really be used carefully.")
+
+    def handle(self, *args, **kwargs):
+        force = kwargs['force']
+
+        if not force:
+            self.stdout.write(self.style.WARNING("This is a dangerous script. "
+                                                 "Please use --force to indicate that you known what you are doing. "
+                                                 "Nothing will be deleted yet."))
+            sleep(5)
+
+        # We need to know who to blame.
+        qs = User.objects.filter(note__alias__normalized_name=Alias.normalize(getpass.getuser()))
+        if not qs.exists():
+            self.stderr.write(self.style.ERROR("I don't know who you are. Please add your linux id as an alias of "
+                                               "your own account."))
+            exit(2)
+        executor = qs.get()
+
+        deleted_users = []
+        deleted = []
+
+        # Don't send mails during the process
+        with override_settings(EMAIL_BACKEND='django.core.mail.backends.dummy.EmailBackend'):
+            for user_id in kwargs['user']:
+                if user_id.isnumeric():
+                    qs = User.objects.filter(pk=int(user_id))
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
+                        continue
+                    user = qs.get()
+                else:
+                    qs = Alias.objects.filter(normalized_name=Alias.normalize(user_id), note__noteuser__isnull=False)
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
+                        continue
+                    user = qs.get().note.user
+
+                with transaction.atomic():
+                    local_deleted = []
+
+                    # Unlock note to enable modifications
+                    if force and not user.note.is_active:
+                        user.note.is_active = True
+                        user.note.save()
+
+                    # Deleting transactions
+                    transactions = Transaction.objects.filter(Q(source=user.note) | Q(destination=user.note)).all()
+                    local_deleted += list(transactions)
+                    for tr in transactions:
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Removing {tr}…")
+                        if force:
+                            tr.delete()
+
+                    # Deleting memberships
+                    memberships = user.memberships.all()
+                    local_deleted += list(memberships)
+                    if kwargs['verbosity'] >= 1:
+                        for membership in memberships:
+                            self.stdout.write(f"Removing {membership}…")
+                    if force:
+                        memberships.delete()
+
+                    # Deleting aliases
+                    alias_set = user.note.alias.all()
+                    local_deleted += list(alias_set)
+                    if kwargs['verbosity'] >= 1:
+                        for alias in alias_set:
+                            self.stdout.write(f"Removing alias {alias}…")
+                    if force:
+                        alias_set.delete()
+
+                    if 'activity' in settings.INSTALLED_APPS:
+                        from activity.models import Guest, Entry
+
+                        # Deleting activity entries
+                        entries = Entry.objects.filter(Q(note=user.note) | Q(guest__inviter=user.note)).all()
+                        local_deleted += list(entries)
+                        if kwargs['verbosity'] >= 1:
+                            for entry in entries:
+                                self.stdout.write(f"Removing {entry}…")
+                        if force:
+                            entries.delete()
+
+                        # Deleting invited guests
+                        guests = Guest.objects.filter(inviter=user.note).all()
+                        local_deleted += list(guests)
+                        if kwargs['verbosity'] >= 1:
+                            for guest in guests:
+                                self.stdout.write(f"Removing guest {guest}…")
+                        if force:
+                            guests.delete()
+
+                    if 'treasury' in settings.INSTALLED_APPS:
+                        from treasury.models import SogeCredit
+
+                        # Deleting soge credit
+                        credits = SogeCredit.objects.filter(user=user).all()
+                        local_deleted += list(credits)
+                        if kwargs['verbosity'] >= 1:
+                            for credit in credits:
+                                self.stdout.write(f"Removing {credit}…")
+                        if force:
+                            credits.delete()
+
+                    # Deleting note
+                    local_deleted.append(user.note)
+                    if force:
+                        user.note.delete()
+
+                    if 'logs' in settings.INSTALLED_APPS:
+                        from logs.models import Changelog
+
+                        # Replace log executors by the runner of the script
+                        Changelog.objects.filter(user=user).update(user=executor)
+
+                    # Deleting profile
+                    local_deleted.append(user.profile)
+                    if force:
+                        user.profile.delete()
+
+                    # Finally deleting user
+                    if force:
+                        user.delete()
+                    local_deleted.append(user)
+
+                    # This script should really not be used.
+                    if not kwargs['doit'] and not input('You are about to delete real user data. '
+                                                        'Are you really sure that it is what you want? [y/N] ')\
+                            .lower().startswith('y'):
+                        self.stdout.write(self.style.ERROR("Aborted."))
+                        exit(1)
+
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(self.style.SUCCESS(f"User {user} deleted."))
+                        deleted_users.append(user)
+                        deleted += local_deleted
+
+        if deleted_users:
+            message = f"Les utilisateur⋅rices {deleted_users} ont été supprimé⋅es par {executor}.\n\n"
+            message += "Ont été supprimés en conséquence les objets suivants :\n\n"
+            for obj in deleted:
+                message += f"{repr(obj)} (pk: {obj.pk})\n"
+            if force and kwargs['doit']:
+                mail_admins("Utilisateur⋅rices supprimés", message)

management/commands/import_account.py

@@ -1,24 +1,22 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
-import psycopg2 as pg
-import psycopg2.extras as pge
 import datetime
 import json
 
-from django.utils.timezone import make_aware, now
+import psycopg2 as pg
+import psycopg2.extras as pge
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from note.models import Note, NoteUser, NoteClub
-from note.models import Alias
 from member.models import Club, Profile
+from note.models import Alias, Note, NoteClub, NoteUser
 
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 M_DURATION = 396
-M_START = datetime.date(2019, 8, 31)
+M_START = datetime.date(2019, 8, 1)
 M_END = datetime.date(2020, 9, 30)
 
 MAP_IDBDE = {
@@ -32,6 +30,11 @@ MAP_IDBDE = {
 # some Aliases have been created in the fixtures
 ALIAS_SET = {a[0] for a in Alias.objects.all().values_list("normalized_name")}
 
+# Some people might loose some aliases due to normalization. We warn them on them.
+LOST_ALIASES = {}
+# In some rare cases, the username might be in conflict with some others. We change them and warn the users.
+CHANGED_USERNAMES = []
+
 note_user_type = ContentType.objects.get(app_label="note", model="noteuser")
 note_club_type = ContentType.objects.get(app_label="note", model="noteclub")
 
@@ -59,6 +62,19 @@ class Command(ImportCommand):
         Every Model has to be manually created, and no magic `.save()`
         function is being called.
         """
+        # Get promotion and date of creation of the account
+        cur.execute("SELECT idbde, MIN(date) AS created_at, MIN(annee) AS promo FROM adhesions"
+                    " GROUP BY idbde ORDER BY promo, created_at;")
+        MAP_IDBDE_PROMOTION = {}
+        for row in cur:
+            MAP_IDBDE_PROMOTION[row["idbde"]] = row
+
+        cur.execute("SELECT * FROM comptes WHERE idbde <= 0 ORDER BY idbde;")
+        for row in cur:
+            note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]])
+            note.balance = row["solde"]
+            note.save()
+
         cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
         pk_club = 3
         pk_user = 1
@@ -71,9 +87,10 @@ class Command(ImportCommand):
             pseudo = row["pseudo"]
             pseudo_norm = Alias.normalize(pseudo)
             self.update_line(idx, n, pseudo)
-            # clean pseudo (normalized pseudo must be unique)
+            # clean pseudo (normalized pseudo must be unique and not empty)
-            if pseudo_norm in ALIAS_SET:
+            if not pseudo_norm or pseudo_norm in ALIAS_SET:
                 pseudo = pseudo + str(row["idbde"])
+                CHANGED_USERNAMES.append((pk_note, row["idbde"], pseudo))
             else:
                 ALIAS_SET.add(pseudo_norm)
             # clean date
@@ -81,9 +98,8 @@ class Command(ImportCommand):
                 "pk": pk_note,
                 "balance": row['solde'],
                 "last_negative": None,
-                "is_active": True,
+                "is_active": not row["bloque"],
-                "display_image": "",
+                "display_image": "pic/default.png",
-                "created_at": now()
             }
             if row["last_negatif"] is not None:
                 note_dict["last_negative"] = make_aware(row["last_negatif"])
@@ -94,14 +110,25 @@ class Command(ImportCommand):
                 else:
                     passwd_nk15 = ''
 
+                # Note account should have no password and be active
+                if int(row["idbde"]) == 3508:
+                    passwd_nk15 = "ipbased$127.0.0.1"
+                    row["bloque"] = False
+
+                if row["idbde"] not in MAP_IDBDE_PROMOTION:
+                    # NK12 bug. Applying default values
+                    MAP_IDBDE_PROMOTION[row["idbde"]] = {"promo": 2014,
+                                                         "created_at": datetime.datetime(2014, 9, 1, 0, 0, 0)}
+
                 obj_dict = {
                     "pk": pk_user,
                     "username": row["pseudo"],
                     "password": passwd_nk15,
-                    "first_name": row["nom"],
+                    "first_name": row["prenom"],
-                    "last_name": row["prenom"],
+                    "last_name": row["nom"],
                     "email": row["mail"],
-                    "is_active": True,  # temporary
+                    "is_active": not row["bloque"],
+                    "date_joined": make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"]),
                 }
                 profile_dict = {
                     "pk": pk_profile,
@@ -111,7 +138,11 @@ class Command(ImportCommand):
                     "paid": row['normalien'],
                     "registration_valid": True,
                     "email_confirmed": True,
+                    "promotion": MAP_IDBDE_PROMOTION[row["idbde"]]["promo"],
+                    "report_frequency": max(row["report_period"], 0),
+                    "last_report": make_aware(row["previous_report_date"]),
                 }
+                note_dict["created_at"] = make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"])
                 note_dict["polymorphic_ctype"] = note_user_type
                 note_user_dict = {
                     "pk": pk_note,
@@ -137,6 +168,7 @@ class Command(ImportCommand):
                     "pk": pk_club,
                     "name": row["pseudo"],
                     "email": row["mail"],
+                    "parent_club_id": 1,   # All clubs depends on BDE by default
                     "membership_duration": M_DURATION,
                     "membership_start": M_START,
                     "membership_end": M_END,
@@ -145,7 +177,7 @@ class Command(ImportCommand):
                 }
                 note_club_dict = {
                     "pk": pk_note,
-                    "club_id": pk_club,
+                    "club_id": pk_club
                 }
                 alias_dict = {
                     "pk": pk_note,
@@ -153,6 +185,7 @@ class Command(ImportCommand):
                     "normalized_name": Alias.normalize(pseudo),
                     "note_id": pk_note
                 }
+                note_dict["created_at"] = make_aware(row["previous_report_date"])  # Not perfect, but good approximation
                 note_dict["polymorphic_ctype"] = note_club_type
                 bulk_mgr.add(Club(**obj_dict),
                              Note(**note_dict),
@@ -177,11 +210,13 @@ class Command(ImportCommand):
         pk_alias = Alias.objects.order_by('-id').first().id + 1
         for idx, row in enumerate(cur):
             alias_name = row["alias"]
-            alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
+            alias_name = (alias_name[:254] + '…') if len(alias_name) > 255 else alias_name
             alias_norm = Alias.normalize(alias_name)
             self.update_line(idx, n, alias_norm)
             # clean pseudo (normalized pseudo must be unique)
-            if alias_norm in ALIAS_SET:
+            if not alias_norm or alias_norm in ALIAS_SET:
+                LOST_ALIASES.setdefault(MAP_IDBDE[row["idbde"]], [])
+                LOST_ALIASES[MAP_IDBDE[row["idbde"]]].append(alias_name)
                 continue
             else:
                 ALIAS_SET.add(alias_norm)
@@ -212,3 +247,7 @@ class Command(ImportCommand):
             filename = kwargs["save"]
             with open(filename, 'w') as fp:
                 json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
+            with open(filename + ".changed-usernames", 'w') as fp:
+                json.dump(CHANGED_USERNAMES, fp, sort_keys=True, indent=2)
+            with open(filename + ".removed-aliases", 'w') as fp:
+                json.dump(LOST_ALIASES, fp, sort_keys=True, indent=2)

management/commands/import_activities.py

@@ -1,29 +1,32 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
 
 import psycopg2 as pg
 import psycopg2.extras as pge
-import datetime
+from activity.models import Activity, ActivityType, Entry, Guest
-import copy
-
-from django.utils.timezone import make_aware
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from activity.models import ActivityType, Activity, Guest, Entry
 from member.models import Club
-from note.models import Note
+from note.models import Note, NoteUser
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_ACTIVITY = dict()
 
 CLUB_RELOU = [
-    0, # BDE
+    0,  # BDE
-    4771, # Kataclist
+    4771,  # Kataclist
-    5162, # Assurance BDE ?!
+    5162,  # Assurance BDE ?!
-    5164, #S & L
+    5164,  # S & L
-    625, #Aspique
+    625,  # Aspique
-    5154, #Frekens
+    5154,  # Frekens
+    3944,  # DiskJok[ENS]
+    5153,  # Monopo[list]
+    2351,  # JdRM
+    2365,  # Pot Vieux
 ]
 
+
 class Command(ImportCommand):
     """
     Import command for Activities Base Data (Comptes, and Aliases)
@@ -35,16 +38,19 @@ class Command(ImportCommand):
         cur.execute("SELECT * FROM activites ORDER by id")
         n = cur.rowcount
         bulk_mgr = BulkCreateManager(chunk_size=chunk)
-        activity_type_id = ActivityType.objects.get(name="Pot").pk  # Need to be fixed manually
+        pot_id = ActivityType.objects.get(name="Pot").pk
+        club_id = ActivityType.objects.get(name="Soirée de club").pk
         kfet = Club.objects.get(name="Kfet")
         pk_activity = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["titre"])
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             note = self.MAP_IDBDE[row["responsable"]]
             if note == 6244:
-                 # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
+                # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
                 note = Note.objects.get(pk=self.MAP_IDBDE[6524])
-                note = note.user_id
+                note = note.id
             organizer = Club.objects.filter(name=row["signature"])
             if organizer.exists():
                 # Try to find the club that organizes the activity.
@@ -56,8 +62,9 @@ class Command(ImportCommand):
                 "pk": pk_activity,
                 "name": row["titre"],
                 "description": row["description"],
-                "activity_type_id": activity_type_id,  # By default Pot
+                "location": row["lieu"],
-                "creater_id": note,
+                "activity_type_id": pot_id if row["liste"] else club_id,
+                "creater_id": NoteUser.objects.get(pk=note).user.id,
                 "organizer_id": organizer.pk,
                 "attendees_club_id": kfet.pk,  # Maybe fix manually
                 "date_start": make_aware(row["debut"]),

management/commands/import_nk15.py

@@ -1,18 +1,37 @@
-#!/usr/env/bin python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
 import subprocess
 
-from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
-class Command(BaseCommand):
+from ._import_utils import ImportCommand
+
+
+class Command(ImportCommand):
     """
     Command for importing the database of NK15.
-    Need to be run by a user with a registered role in postgres for the database nk15. 
+    Need to be run by a user with a registered role in postgres for the database nk15.
     """
 
     def handle(self, *args, **kwargs):
         subprocess.call("./apps/scripts/shell/tabularasa")
-        call_command('import_account', alias=True, chunk=1000, save = "map.json")
+
-        call_command('import_activities', chunk=100, map="map.json")
+        kwargs["alias"] = True
-        call_command('import_transaction', chunk=10000, buttons=True, map="map.json")
+        kwargs["chunk"] = 1000
-#
+        kwargs["save"] = "map.json"
+        call_command('import_account', **kwargs)
+
+        del kwargs["alias"]
+        del kwargs["save"]
+        kwargs["chunk"] = 100
+        kwargs["map"] = "map.json"
+        call_command('import_activities', **kwargs)
+
+        kwargs["chunk"] = 10000
+        kwargs["map"] = "map.json"
+        kwargs["buttons"] = True
+        call_command('import_transaction', **kwargs)
+
+        call_command('make_su','-sS', 'Coq', 'erdnaxe', 'Krokmou', 'PAC', 'Pollion', 'TLinux', 'ÿnérant')
+        call_command('syncsql')

management/commands/import_transaction.py

@@ -1,40 +1,55 @@
-#!/usr/bin/env python3
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import copy
+import datetime
 import re
+
+import pytz
 import psycopg2 as pg
 import psycopg2.extras as pge
-import pytz
+from activity.models import Entry, GuestTransaction
-import datetime
+from django.contrib.auth.models import User
-import copy
+from django.contrib.contenttypes.models import ContentType
-
-from django.utils.timezone import make_aware
 from django.db import transaction
+from django.utils.timezone import make_aware
+from member.models import Membership
+from note.models import (MembershipTransaction, Note, NoteClub,
+                         RecurrentTransaction, SpecialTransaction,
+                         TemplateCategory, Transaction, TransactionTemplate)
+from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy
 
-from note.models import (TemplateCategory,
+from ._import_utils import BulkCreateManager, ImportCommand, timed
-                         TransactionTemplate,
-                         Transaction,
-                         RecurrentTransaction,
-                         SpecialTransaction
-                         )
-from note.models import Note, NoteClub
-from activity.models import Guest, GuestTransaction
 
-from member.models import Membership, MembershipTransaction
+MAP_TRANSACTION = dict()
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+MAP_REMITTANCE = dict()
 
+# from member/fixtures/initial
 BDE_PK = 1
 KFET_PK = 2
+
+# from note/fixtures/initial
 NOTE_SPECIAL_CODE = {
     "espèce": 1,
     "carte": 2,
     "chèque": 3,
     "virement": 4,
 }
+# from permission/fixtures/initial
+BDE_ROLE_PK = 1
+KFET_ROLE_PK = 2
 
+CT = {
+    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
+    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
+    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
+    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
+}
 
 def get_date_end(date_start):
     date_end = copy.deepcopy(date_start)
-    if date_start.month > 8:
+    if date_start.month >= 8:
-        date_end = date_start.replace(year=date_start.year+1)
+        date_end = date_start.replace(year=date_start.year + 1)
     date_end = date_end.replace(month=9, day=30)
     return date_end
 
@@ -47,9 +62,11 @@ class Command(ImportCommand):
     def add_arguments(self, parser):
         parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
+        parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
+                                                                                       "don't save them")
 
     @timed
-    def import_buttons(self, cur, chunk_size):
+    def import_buttons(self, cur, chunk_size, import_buttons):
         self.categories = dict()
         self.buttons = dict()
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
@@ -58,7 +75,7 @@ class Command(ImportCommand):
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["label"])
             if row["categorie"] not in self.categories:
-                cat = TemplateCategory.objects.create(name=row["categorie"])
+                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
                 cat.save()
                 self.categories[row["categorie"]] = cat.pk
             obj_dict = {
@@ -72,28 +89,30 @@ class Command(ImportCommand):
             }
             if row["label"] in self.buttons:
                 obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
-            bulk_mgr.add(TransactionTemplate(**obj_dict))
+            if import_buttons:
+                bulk_mgr.add(TransactionTemplate(**obj_dict))
             self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
         bulk_mgr.done()
 
     def _basic_transaction(self, row, obj_dict, child_dict):
         if len(row["description"]) > 255:
-            obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
+            obj_dict["reason"] = obj_dict["reason"][:252] + "…)"
         return obj_dict, None, None
 
     def _template_transaction(self, row, obj_dict, child_dict):
-        if self.categories.get(row["categorie"]):
+        if self.buttons.get(row["description"]):
-            child_dict["category_id"] = self.categories[row["categorie"]]
+            child_dict["template_id"] = self.buttons[row["description"]][0]
+        # elif self.categories.get(row["categorie"]):
+        #     child_dict["category_id"] = self.categories[row["categorie"]]
         elif "WEI" in row["description"]:
             return obj_dict, None, None
-        elif self.buttons.get(row["description"]):
-            child_dict["category_id"] = self.buttons[row["description"]][1]
-            child_dict["template_id"] = self.buttons[row["description"]][0]
         else:
             return obj_dict, None, None
+        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
         return obj_dict, child_dict, RecurrentTransaction
 
     def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
+        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
         obj_dict2 = obj_dict.copy()
         child_dict2 = child_dict.copy()
         child_dict2["membership_id"] = pk_membership
@@ -103,6 +122,7 @@ class Command(ImportCommand):
     def _special_transaction(self, row, obj_dict, child_dict):
         # Some transaction uses BDE (idbde=0) as source or destination,
         # lets fix that.
+        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
         field_id = "source_id" if row["type"] == "crédit" else "destination_id"
         if "espèce" in row["description"]:
             obj_dict[field_id] = 1
@@ -125,25 +145,30 @@ class Command(ImportCommand):
         return obj_dict, child_dict, SpecialTransaction
 
     def _guest_transaction(self, row, obj_dict, child_dict):
-        # Currently GuestTransaction is related to a Guest.
+        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
-        # This is not ideal and should be change to the Entry of this Guest.
         m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
         if m:
+            activity_name = m.group(1)
             first_name, last_name = m.group(2), m.group(3)
-            guest_id = Guest.objects.filter(first_name__iexact=first_name,
+            if first_name == "Marion" and last_name == "Bizu Pose":
-                                           last_name__iexact=last_name).first().pk
+                first_name, last_name = "Marion Bizu", "Pose"
-            child_dict["guest_id"] = guest_id
+            entry_id = Entry.objects.filter(
+                    activity__name__iexact=activity_name,
+                    guest__first_name__iexact=first_name,
+                    guest__last_name__iexact=last_name,
+            ).first().pk
+            child_dict["entry_id"] = entry_id
         else:
-            raise(f"Guest not Found {row['id']} {first_name}, last_name")
+            raise Exception(f"Guest not Found {row['id']} first_name, last_name")
 
         return obj_dict, child_dict, GuestTransaction
 
     @timed
     @transaction.atomic
-    def import_transaction(self, cur, chunk_size, idmin):
+    def import_transaction(self, cur, chunk_size, idmin, save=True):
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute(
-            f"SELECT t.date AS transac_date, t.type, t.emetteur,\
+            f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
               t.destinataire,t.quantite, t.montant, t.description,\
               t.valide, t.cantinvalidate, t.categorie, \
               a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
@@ -154,8 +179,21 @@ class Command(ImportCommand):
         n = cur.rowcount
         pk_membership = 1
         pk_transaction = 1
+        kfet_balance = 0
         for idx, row in enumerate(cur):
-            self.update_line(idx, n, row["description"])
+            if save or idx % chunk_size == 0:
+                self.update_line(idx, n, row["description"])
+
+            MAP_TRANSACTION[row["id"]] = pk_transaction
+
+            if not save:
+                pk_transaction += 1
+                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
+                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
+                    if not isinstance(note, NoteClub):
+                        pk_transaction += 1
+                continue
+
             try:
                 date = make_aware(row["transac_date"])
             except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
@@ -165,35 +203,46 @@ class Command(ImportCommand):
             obj_dict = {
                 "pk": pk_transaction,
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
+                "polymorphic_ctype": None,
                 "source_id": self.MAP_IDBDE[row["emetteur"]],
-                "created_at": date,
                 "amount": row["montant"],
+                "created_at": date,
+                "destination_alias": "",
+                "invalidity_reason": "",
                 "quantity": row["quantite"],
                 "reason": row["description"],
+                "source_alias": "",
                 "valid": row["valide"],
             }
+            if len(obj_dict["reason"]) > 255:
+                obj_dict["reason"] = obj_dict["reason"][:254] + "…"
             # for child transaction Models
-            child_dict = {"pk": obj_dict["pk"]}
+            child_dict = {"pk": pk_transaction}
             ttype = row["type"]
+            # Membership transaction detection and import
             if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                 note = Note.objects.get(pk=obj_dict["source_id"])
                 if isinstance(note, NoteClub):
-                    child_transaction = None
+                    child_transaction = None    # don't bother register clubs
                 else:
                     user_id = note.user_id
                     montant = obj_dict["amount"]
-                    obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    (obj_dict0,
+                     child_dict0,
+                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
+                    obj_dict0["destination_id"] = 6  # Kfet note id
                     bde_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": KFET_PK,
+                        "club_id": BDE_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": min(500, montant)
                     }
                     pk_membership += 1
                     pk_transaction += 1
-                    obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    obj_dict, child_dict, child_transaction =\
+                        self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                     # Kfet membership
                     # BDE Membership
                     obj_dict["pk"] = pk_transaction
@@ -201,23 +250,24 @@ class Command(ImportCommand):
                     kfet_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": BDE_PK,
+                        "club_id": KFET_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": max(montant - 500, 0),
                     }
                     obj_dict0["amount"] = bde_dict["fee"]
                     obj_dict["amount"] = kfet_dict["fee"]
+                    kfet_balance += kfet_dict["fee"]
                     # BDE membership Transaction is inserted before the Kfet membershipTransaction
                     pk_membership += 1
                     pk_transaction += 1
                     bulk_mgr.add(
+                        Membership(**bde_dict),
+                        Membership(**kfet_dict),
                         Transaction(**obj_dict0),
                         child_transaction(**child_dict0),
                         Transaction(**obj_dict),
                         child_transaction(**child_dict),
-                        Membership(**bde_dict),
-                        Membership(**kfet_dict),
                     )
                     continue
             elif ttype == "bouton":
@@ -226,16 +276,121 @@ class Command(ImportCommand):
                 obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
             elif ttype == "invitation":
                 obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
-            if ttype == "don" or ttype == "transfert":
+            elif ttype == "don" or ttype == "transfert":
                 obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
             else:
                 child_transaction = None
                 # create base transaction object and typed one
             bulk_mgr.add(Transaction(**obj_dict))
             if child_transaction is not None:
+                child_dict.update(obj_dict)
                 bulk_mgr.add(child_transaction(**child_dict))
             pk_transaction += 1
         bulk_mgr.done()
+
+        # Update the balance of the BDE and the Kfet club
+        note_bde = NoteClub.objects.get(pk=5)
+        note_kfet = NoteClub.objects.get(pk=6)
+        note_bde.balance -= kfet_balance
+        note_kfet.balance += kfet_balance
+        note_bde.save()
+        note_kfet.save()
+
+    @timed
+    def set_roles(self):
+        bulk_mgr = BulkCreateManager(chunk_size=10000)
+        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
+        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
+        n = len(bde_membership_ids)
+        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
+            self.update_line(idx, n, str(idx))
+            bulk_mgr.add(
+                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
+                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
+            )
+            bulk_mgr.done()
+
+        # Note account has a different treatment
+        for m in Membership.objects.filter(user_username="note").all():
+            m.date_end = "3142-12-12"
+            m.roles.set([20])   # PC Kfet role
+            m.save()
+
+    @timed
+    @transaction.atomic
+    def import_remittances(self, cur, chunk_size):
+        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
+        cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
+        n = cur.rowcount
+        pk_remittance = 1
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["commentaire"])
+
+            MAP_REMITTANCE[row["id"]] = pk_remittance
+
+            remittance_dict = {
+                "pk": pk_remittance,
+                "date": make_aware(row["date"]),
+                "remittance_type_id": 1,    # Only Bank checks are supported in NK15
+                "comment": row["commentaire"],
+                "closed": row["close"],
+            }
+            bulk_mgr.add(Remittance(**remittance_dict))
+
+            pk_remittance += 1
+
+        bulk_mgr.done()
+
+    @timed
+    def import_checks(self, cur):
+        cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
+                    "FROM cheques ORDER BY id;")
+        n = cur.rowcount
+
+        for idx, row in enumerate(cur):
+            self.update_line(idx, n, row["nom"])
+
+            if not row["idremise"]:
+                continue
+
+            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
+            tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
+            tr.save()
+
+            tr = tr.transaction
+            tr.last_name = row["nom"]
+            tr.first_name = row["prenom"]
+            tr.bank = row["banque"]
+            try:
+                tr.save()
+            except:
+                print("Failed to save row: " + str(row))
+
+    @timed
+    def import_soge_credits(self):
+        users = User.objects.filter(profile__registration_valid=True).order_by('pk')
+        n = users.count()
+        for idx, user in enumerate(users.all()):
+            self.update_line(idx, n, user.username)
+            soge_credit_transaction = SpecialTransaction.objects.filter(
+                reason__icontains="crédit sogé",
+                destination_id=user.note.id,
+            )
+            if soge_credit_transaction.exists():
+                soge_credit_transaction = soge_credit_transaction.get()
+                soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
+                memberships = Membership.objects.filter(
+                    user=user,
+                    club_id__in=[BDE_PK, KFET_PK],
+                    date_start__lte=soge_credit_transaction.created_at,
+                    date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
+                ).all()
+                for membership in memberships:
+                    soge_credit.transactions.add(membership.transaction)
+                soge_credit.save()
+
+
     @timed
     def handle(self, *args, **kwargs):
         # default args, provided by ImportCommand.
@@ -246,5 +401,10 @@ class Command(ImportCommand):
 
         if kwargs["map"]:
             self.load_map(kwargs["map"])
-        self.import_buttons(cur, kwargs["chunk"])
+        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
-        self.import_transaction(cur, kwargs["chunk"], 0)
+        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
+        if not kwargs["nosave"]:
+            self.set_roles()
+        self.import_remittances(cur, kwargs["chunk"])
+        self.import_checks(cur)
+        self.import_soge_credits()

management/commands/make_su.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.core.management.base import BaseCommand
@@ -16,7 +16,11 @@ class Command(BaseCommand):
             user = User.objects.get(username=uname)
             user.is_active = True
             if kwargs['STAFF']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to staff users…")
                 user.is_staff = True
             if kwargs['SUPER']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to superusers…")
                 user.is_superuser = True
             user.save()

management/commands/merge_club.py

@@ -0,0 +1,302 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import getpass
+from time import sleep
+from copy import copy
+
+from django.conf import settings
+from django.core.mail import mail_admins
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.db.models import Q
+from django.test import override_settings
+from note.models import Alias, Transaction, TransactionTemplate
+from member.models import Club, Membership
+
+
+class Command(BaseCommand):
+    """
+    This script is used to merge clubs.
+    THIS IS DANGEROUS SCRIPT, use it only if you know what you do !!!
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument('--fake_club', '-c', type=str, nargs='+', help="Club id to merge and delete.")
+        parser.add_argument('--true_club', '-C', type=str, help="Club id will not be deleted.")
+        parser.add_argument('--force', '-f', action='store_true',
+                            help="Force the script to have low verbosity.")
+        parser.add_argument('--doit', '-d', action='store_true',
+                            help="Don't ask for a final confirmation and commit modification. "
+                                 "This option should really be used carefully.")
+
+    def handle(self, *args, **kwargs):
+        force = kwargs['force']
+
+        if not force:
+            self.stdout.write(self.style.WARNING("This is a dangerous script. "
+                                                 "Please use --force to indicate that you known what you are doing. "
+                                                 "Nothing will be deleted yet."))
+            sleep(5)
+
+        # We need to know who to blame.
+        qs = User.objects.filter(note__alias__normalized_name=Alias.normalize(getpass.getuser()))
+        if not qs.exists():
+            self.stderr.write(self.style.ERROR("I don't know who you are. Please add your linux id as an alias of "
+                                               "your own account."))
+            exit(2)
+        executor = qs.get()
+
+        deleted_clubs = []
+        deleted = []
+        created = []
+        edited = []
+
+        # Don't send mails during the process
+        with override_settings(EMAIL_BACKEND='django.core.mail.backends.dummy.EmailBackend'):
+            true_club_id = kwargs['true_club']
+            if true_club_id.isnumeric():
+                qs = Club.objects.filter(pk=int(true_club_id))
+                if not qs.exists():
+                    self.stderr.write(self.style.WARNING(f"Club {true_club_id} was not found. Aborted…"))
+                    exit(2)
+                true_club = qs.get()
+            else:
+                qs = Alias.objects.filter(normalized_name=Alias.normalize(true_club_id), note__noteclub__isnull=False)
+                if not qs.exists():
+                    self.stderr.write(self.style.WARNING(f"Club {true_club_id} was not found. Aborted…"))
+                    exit(2)
+                true_club = qs.get().note.club
+
+            fake_clubs = []
+            for fake_club_id in kwargs['fake_club']:
+                if fake_club_id.isnumeric():
+                    qs = Club.objects.filter(pk=int(fake_club_id))
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"Club {fake_club_id} was not found. Ignoring…"))
+                        continue
+                    fake_clubs.append(qs.get())
+                else:
+                    qs = Alias.objects.filter(normalized_name=Alias.normalize(fake_club_id), note__noteclub__isnull=False)
+                    if not qs.exists():
+                        self.stderr.write(self.style.WARNING(f"Club {fake_club_id} was not found. Ignoring…"))
+                        continue
+                    fake_clubs.append(qs.get().note.club)
+
+            clubs = fake_clubs.copy()
+            clubs.append(true_club)
+            for club in fake_clubs:
+                children = Club.objects.filter(parent_club=club)
+                for child in children:
+                    if child not in fake_clubs:
+                        self.stderr.write(self.style.ERROR(f"Club {club} has child club {child} which are not selected for merge. Aborted."))
+                        exit(1)
+
+            with transaction.atomic():
+                local_deleted = []
+                local_created = []
+                local_edited = []
+
+                # Unlock note to enable modifications
+                for club in clubs:
+                    if force and not club.note.is_active:
+                        club.note.is_active = True
+                        club.note.save()
+
+                # Deleting objects linked to fake_club and true_club
+
+                # Deleting transactions
+                # We delete transaction :
+                #   fake_club_i <-> fake_club_j
+                #   fake_club_i <-> true_club
+                transactions = Transaction.objects.filter(Q(source__noteclub__club__in=clubs)
+                                                          & Q(destination__noteclub__club__in=clubs)).all()
+                local_deleted += list(transactions)
+                for tr in transactions:
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Removing {tr}…")
+                    if force:
+                        tr.delete()
+
+                # Merge buttons
+                buttons = TransactionTemplate.objects.filter(destination__club__in=fake_clubs)
+                local_edited += list(buttons)
+                for b in buttons:
+                    b.destination = true_club.note
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Edit {b}")
+                    if force:
+                        b.save()
+
+                # Merge transactions
+                transactions = Transaction.objects.filter(source__noteclub__club__in=fake_clubs)
+                local_deleted += list(transactions)
+                for tr in transactions:
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Removing {tr}…")
+                    tr_merge = copy(tr)
+                    tr_merge.pk = None
+                    tr_merge.source = true_club.note
+                    local_created.append(tr_merge)
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Creating {tr_merge}…")
+                    if force:
+                        if not tr.destination.is_active:
+                            tr.destination.is_active = True
+                            tr.destination.save()
+                            tr.delete()
+                            tr_merge.save()
+                            tr.destination.is_active = False
+                            tr.destination.save()
+                        else:
+                            tr.delete()
+                            tr_merge.save()
+                transactions = Transaction.objects.filter(destination__noteclub__club__in=fake_clubs)
+                local_deleted += list(transactions)
+                for tr in transactions:
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Removing {tr}…")
+                    tr_merge = copy(tr)
+                    tr_merge.pk = None
+                    tr_merge.destination = true_club.note
+                    local_created.append(tr_merge)
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Creating {tr_merge}…")
+                    if force:
+                        if not tr.source.is_active:
+                            tr.source.is_active = True
+                            tr.source.save()
+                            tr.delete()
+                            tr_merge.save()
+                            tr.source.is_active = False
+                            tr.source.save()
+                        else:
+                            tr.delete()
+                            tr_merge.save()
+                if 'permission' in settings.INSTALLED_APPS:
+                    from permission.models import Role
+                    r = Role.objects.filter(for_club__in=fake_clubs)
+                    for role in r:
+                        role.for_club = true_club
+                        local_edited.append(role)
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Edit {role}…")
+                        if force:
+                            role.save()
+
+                # Merge memberships
+                for club in fake_clubs:
+                    memberships = Membership.objects.filter(club=club)
+                    local_edited += list(memberships)
+                    for membership in memberships:
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Edit {membership}…")
+                        if force:
+                            membership.club = true_club
+                            membership.save()
+
+                # Merging aliases
+                alias_list = []
+                for fake_club in fake_clubs:
+                    alias_list += list(fake_club.note.alias.all())
+                local_deleted += alias_list
+                for alias in alias_list:
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Removing alias {alias}…")
+                    alias_merge = alias
+                    alias_merge.note = true_club.note
+                    local_created.append(alias_merge)
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Creating alias {alias_merge}…")
+                    if force:
+                        alias.delete()
+                        alias_merge.save()
+
+                if 'activity' in settings.INSTALLED_APPS:
+                    from activity.models import Activity
+
+                    # Merging activities
+                    activities = Activity.objects.filter(organizer__in=fake_clubs)
+                    for act in activities:
+                        act.organizer = true_club
+                        local_edited.append(act)
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Edit {act}…")
+                        if force:
+                            act.save()
+                    activities = Activity.objects.filter(attendees_club__in=fake_clubs)
+                    for act in activities:
+                        act.attendees_club = true_club
+                        local_edited.append(act)
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Edit {act}…")
+                        if force:
+                            act.save()
+
+                if 'food' in settings.INSTALLED_APPS:
+                    from food.models import Food
+                    foods = Food.objects.filter(owner__in=fake_clubs)
+                    for f in foods:
+                        f.owner = true_club
+                        local_edited.append(f)
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Edit {f}…")
+                        if force:
+                            f.save()
+
+                if 'wrapped' in settings.INSTALLED_APPS:
+                    from wrapped.models import Wrapped
+                    wraps = Wrapped.objects.filter(note__noteclub__club__in=fake_clubs)
+                    local_deleted += list(wraps)
+                    for w in wraps:
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Remove {w}…")
+                        if force:
+                            w.delete()
+
+                # Deleting note
+                for club in fake_clubs:
+                    local_deleted.append(club.note)
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Remove note of {club}…")
+                    if force:
+                        club.note.delete()
+
+                # Finally deleting user
+                for club in fake_clubs:
+                    local_deleted.append(club)
+                    if kwargs['verbosity'] >= 1:
+                        self.stdout.write(f"Remove {club}…")
+                    if force:
+                        club.delete()
+
+                # This script should really not be used.
+                if not kwargs['doit'] and not input('You are about to delete real user data. '
+                                                    'Are you really sure that it is what you want? [y/N] ')\
+                        .lower().startswith('y'):
+                    self.stdout.write(self.style.ERROR("Aborted."))
+                    exit(1)
+
+                if kwargs['verbosity'] >= 1:
+                    for club in fake_clubs:
+                        self.stdout.write(self.style.SUCCESS(f"Club {club} deleted and merge in {true_club}."))
+                        deleted_clubs.append(clubs)
+                    self.stdout.write(self.style.WARNING("There are problems with balance of inactive note impact by the fusion, run './manage.py check_consistency -a -f' to fix"))
+                    deleted += local_deleted
+                    created += local_created
+                    edited += local_edited
+
+        if deleted_clubs:
+            message = f"Les clubs {deleted_clubs} ont été supprimé⋅es pour être fusionné dans le club {true_club} par {executor}.\n\n"
+            message += "Ont été supprimés en conséquence les objets suivants :\n\n"
+            for obj in deleted:
+                message += f"{repr(obj)} (pk: {obj.pk})\n"
+            message += "\n\nOnt été créés en conséquence les objects suivants :\n\n"
+            for obj in created:
+                message += f"{repr(obj)} (pk: {obj.pk})\n"
+            message += "\n\nOnt été édités en conséquence les objects suivants :\n\n"
+            for obj in edited:
+                message += f"{repr(obj)} (pk: {obj.pk})\n"
+            if force and kwargs['doit']:
+                mail_admins("Clubs fusionnés", message)

management/commands/refresh_activities.py

@@ -0,0 +1,180 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import os
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from bs4 import BeautifulSoup
+from activity.models import Activity
+from django.core.management import BaseCommand
+from django.urls import reverse
+from django.utils import timezone
+
+
+class Command(BaseCommand):
+    acl_header = "#acl NoteKfet2020:read,write,admin All:read Default\n"
+
+    warning_header = """## NE PAS ÉDITER CETTE PAGE MANUELLEMENT
+## ELLE EST GÉNÉRÉE AUTOMATIQUEMENT PAR LA NOTE KFET 2020
+## Adapté par [[WikiYnerant|ÿnérant]] du script de by 20-100, largement inspiré de la version de Barbichu.
+"""
+
+    intro_generic = """ * Elle est générée automatiquement par la [[NoteKfet/NoteKfet2020|Note Kfet 2020]]
+ * Ne pas éditer cette page manuellement, toute modification sera annulée automatiquement.
+ * Pour annoncer un nouvel événement, rendez-vous sur {activities_url}
+
+""".format(activities_url="https://" + os.getenv("NOTE_URL", "") + reverse("activity:activity_list"))
+
+    @staticmethod
+    def connection(url):
+        """Se logue sur le wiki et renvoie le cookie de session"""
+        parameters = {
+            'action': 'login',
+            'login': 'Connexion',
+            'name': os.getenv("WIKI_USER", "NoteKfet2020"),
+            'password': os.getenv("WIKI_PASSWORD"),
+        }
+        # Il faut encoder ça proprement
+        data = urlencode(parameters).encode("utf-8")
+        request = Request(url, data)
+        # La requête est envoyée en HTTP POST
+        response = urlopen(request)
+        # a priori la page elle-même je m'en carre…
+        response.read(2)
+        # …ce qui m'intéresse, c'est le cookie qu'elle me file
+        cookie = response.headers['set-cookie']
+        return cookie
+
+    @staticmethod
+    def get_edition_ticket(url, cookie):
+        """Récupère le ticket d'édition de la page"""
+        # On crée la requête d'édition…
+        suffix = "?action=edit&editor=text"
+        request = Request(url + suffix)
+        # …avec le cookie
+        request.add_header("Cookie", cookie)
+        # On l'envoie
+        pagecontent = urlopen(request)
+        html = pagecontent.read()
+        soup = BeautifulSoup(html, features="lxml")
+        # On va chercher le formulaire
+        form = soup.find(name="form", attrs={"id": "editor"})
+        # On récupère le ticket dedans
+        ticket = soup.find(name="input", attrs={"name": "ticket"})
+        return ticket["value"]
+
+    @staticmethod
+    def edit_wiki(page, content, comment=''):
+        """Modifie une page du wiki"""
+        url = "https://wiki.crans.org/" + page
+
+        # On se connecte et on récupère le cookie de session
+        cookie = Command.connection(url)
+        # On demande l'édition et on récupère le ticket d'édition de la page
+        ticket = Command.get_edition_ticket(url, cookie)
+        # On construit la requête
+        data = {
+            'button_save': 'Enregistrer les modifications',
+            'category': '',
+            'comment': comment.encode("utf-8"),
+            'savetext': content.encode("utf-8"),
+            'action': 'edit',
+            'ticket': ticket
+        }
+        request = Request(url, urlencode(data).encode("utf-8"))
+        request.add_header("Cookie", cookie)
+        # On la poste
+        urlopen(request)
+
+    @staticmethod
+    def format_activity(act, raw=True):
+        """Wiki-formate une activité, pour le calendrier raw si ``raw``, pour le human-readable sinon."""
+        if raw:
+            return """== {title} ==
+ start:: {start}
+ end:: {end}
+ description:: {description} -- {club}
+ location:: {location}
+""".format(
+                title=act.name,
+                start=timezone.localtime(act.date_start).strftime("%Y-%m-%d %H:%M"),
+                end=timezone.localtime(act.date_end).strftime("%Y-%m-%d %H:%M"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
+                club=act.organizer.name,
+                location=act.location,
+            )
+        else:
+            return "|| {start} || {title} || {description} || {club} || {location} ||".format(
+                title=act.name,
+                start=timezone.localtime(act.date_start).strftime("%d/%m/%Y"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
+                club=act.organizer.name,
+                location=act.location,
+            )
+
+    @staticmethod
+    def get_raw_page():
+        page = "VieBde/PlanningSoirees/LeCalendrier"
+        header = Command.acl_header + Command.warning_header
+        header += """= Introduction =
+
+ * Cette page a pour but de recenser les activités BDE afin d'être signalées sur le calendrier de la
+ [[PageAccueil|page d'accueil]] du wiki.
+"""
+        header += Command.intro_generic
+        body = "\n".join(Command.format_activity(activity) for activity in Activity.objects.filter(valid=True)
+                         .order_by('-date_start').all())
+        footer = "\n----\nCatégorieCalendrierCampus"
+        return page, header + body + footer
+
+    @staticmethod
+    def get_human_readable_page():
+        page = "VieBde/PlanningSoirees"
+        header = Command.acl_header + Command.warning_header
+        header += """= Planning de soirées =
+== Introduction ==
+ * Cette page est destinée à accueillir le planning des soirées BDE.
+"""
+        header += Command.intro_generic + "\n"
+        body = """== Planning des activités à venir ==
+||'''Date'''||'''Titre'''||'''Description'''||'''Par''' ||'''Lieu'''||
+"""
+        body += "\n".join(Command.format_activity(activity, False) for activity in Activity.objects
+                          .filter(valid=True, date_end__gte=timezone.now()).order_by('-date_start').all())
+        body += """\n\n== Planning des activités passées ==
+||'''Date'''||'''Titre'''||'''Description'''||'''Par'''||'''Lieu'''||
+"""
+        body += "\n".join(Command.format_activity(activity, False) for activity in Activity.objects
+                          .filter(valid=True, date_end__lt=timezone.now()).order_by('-date_start').all())
+        return page, header + body
+
+    @staticmethod
+    def refresh_raw_wiki_page(comment="refresh", print_stdout=False, edit_wiki=False):
+        page, content = Command.get_raw_page()
+        if print_stdout:
+            print(content)
+        if edit_wiki:
+            Command.edit_wiki(page, content, comment)
+
+    @staticmethod
+    def refresh_human_readable_wiki_page(comment="refresh", print_stdout=False, edit_wiki=False):
+        page, content = Command.get_human_readable_page()
+        if print_stdout:
+            print(content)
+        if edit_wiki:
+            Command.edit_wiki(page, content, comment)
+
+    def add_arguments(self, parser):
+        parser.add_argument("--human", "-H", action="store_true", help="Save human readable page")
+        parser.add_argument("--raw", "-r", action="store_true", help="Save raw page, for the calendar")
+        parser.add_argument("--comment", "-c", action="store", type=str, default="", help="Comment of the modification")
+        parser.add_argument("--stdout", "-o", action="store_true", help="Render the wiki page in stdout")
+        parser.add_argument("--wiki", "-w", action="store_true", help="Send modifications to the wiki")
+
+    def handle(self, *args, **options):
+        if options["raw"]:
+            Command.refresh_raw_wiki_page(options["comment"], options["stdout"], options["wiki"])
+        if options["human"]:
+            Command.refresh_human_readable_wiki_page(options["comment"], options["stdout"], options["wiki"])
+

management/commands/refresh_highlighted_buttons.py

@@ -0,0 +1,29 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.db.models import Count
+from django.utils import timezone
+from note.models import RecurrentTransaction, TransactionTemplate
+
+
+class Command(BaseCommand):
+    """
+    Command to add the ten most used buttons of the past month to the highlighted buttons.
+    """
+    def handle(self, *args, **kwargs):
+        queryset = RecurrentTransaction.objects.filter(
+            template__display=True,
+            valid=True,
+            created_at__gte=timezone.now() - timedelta(days=30),
+        ).values("template").annotate(transaction_count=Count("template")).order_by("-transaction_count")[:10]
+        for d in queryset.all():
+            button_id = d["template"]
+            button = TransactionTemplate.objects.get(pk=button_id)
+            if kwargs['verbosity'] > 0:
+                self.stdout.write(self.style.WARNING("Highlight button {name} ({count:d} transactions)..."
+                                                     .format(name=button.name, count=d["transaction_count"])))
+            button.highlighted = True
+            button.save()

management/commands/send_mail_for_food.py

@@ -0,0 +1,41 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date, timedelta
+
+from django.core.mail import send_mail
+from django.core.management import BaseCommand
+from django.template.loader import render_to_string
+from django.utils.translation import activate
+from food.models import Food
+from member.models import Club
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--report", "-r", action='store_true', help="Report the list of food to GCKs")
+        parser.add_argument("--club", "-c", action='store_true', help="Report the list of food to club")
+
+    def handle(self, *args, **options):
+        activate('fr')
+        
+        foods = Food.objects.filter(end_of_life='').order_by('expiry_date').distinct().all()
+
+        if options["report"]:
+            plain_text = render_to_string("scripts/food_report.txt", context=dict(foods=foods))
+            html = render_to_string("scripts/food_report.html", context=dict(foods=foods))
+            send_mail("[Note Kfet] Liste de la nourriture à la Kfet", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                      recipient_list=["respo-info.bde@lists.crans.org", "gck.bde@lists.crans.org"],
+                      html_message=html)
+
+        if options["club"]:
+            for club in Club.objects.all():
+                if Food.objects.filter(end_of_life='', owner=club).count() > 0:
+                    plain_text = render_to_string("scripts/food_report.txt",
+                                                  context=dict(foods=foods.filter(owner=club)))
+                    html = render_to_string("scripts/food_report.html",
+                                            context=dict(foods=foods.filter(owner=club)))
+                    send_mail("[Note Kfet] Liste de la nourriture de votre club", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                              recipient_list=[club.email],
+                              html_message=html)
+

management/commands/send_mail_to_negative_balances.py

@@ -0,0 +1,48 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date, timedelta
+
+from django.core.mail import send_mail
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils.translation import activate
+from note.models import Note
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--spam", "-s", action='store_true', help="Spam negative users")
+        parser.add_argument("--report", "-r", action='store_true', help="Report the list of negative users to admins")
+        parser.add_argument("--negative-amount", "-n", action='store', type=int, default=1000,
+                            help="Maximum amount to be considered as very negative (inclusive)")
+        parser.add_argument("--add-years", "-y", action='store', type=int, default=0,
+                            help="Add also people that have a negative balance since N years "
+                                 "(default to only active members)")
+
+    def handle(self, *args, **options):
+        activate('fr')
+
+        if options['negative_amount'] == 0:
+            # Don't log empty notes
+            options['negative_amount'] = 1
+
+        notes = Note.objects.filter(
+            Q(noteuser__user__memberships__date_end__gte=
+              date.today() - timedelta(days=int(365.25 * options['add_years'])))
+            | (Q(noteclub__isnull=False) & ~Q(noteclub__club__name__icontains='- BDE')),
+            balance__lte=-options["negative_amount"],
+            is_active=True, 
+        ).order_by('balance').distinct().all()
+
+        if options["spam"]:
+            for note in notes:
+                note.send_mail_negative_balance()
+
+        if options["report"]:
+            plain_text = render_to_string("note/mails/negative_notes_report.txt", context=dict(notes=notes))
+            html = render_to_string("note/mails/negative_notes_report.html", context=dict(notes=notes))
+            send_mail("[Note Kfet] Liste des négatifs", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                      recipient_list=["respo-info.bde@lists.crans.org", "tresorerie.bde@lists.crans.org"],
+                      html_message=html)

management/commands/send_reports.py

@@ -0,0 +1,61 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils import timezone
+from django.utils.translation import activate
+from note.models import NoteUser, Transaction
+from note.tables import HistoryTable
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--notes', '-n', type=int, nargs='+', help='Select note ids')
+        parser.add_argument('--debug', '-d', action='store_true', help='Debug mode, print mails in stdout')
+
+    def handle(self, *args, **options):
+        activate('fr')
+        if 'notes' in options and options['notes']:
+            notes = NoteUser.objects.filter(pk__in=options['notes']).all()
+        else:
+            notes = NoteUser.objects.filter(
+                user__memberships__date_end__gte=timezone.now(),
+                user__profile__report_frequency__gt=0,
+            ).distinct().all()
+        for note in notes:
+            now = timezone.now()
+            last_report = note.user.profile.last_report
+            delta = now.date() - last_report.date()
+            if delta.days < note.user.profile.report_frequency:
+                continue
+            if not options["debug"]:
+                note.user.profile.last_report = now
+                note.user.profile.save()
+            last_transactions = Transaction.objects.filter(
+                Q(source=note) | Q(destination=note),
+                created_at__gte=last_report,
+            ).order_by("created_at").all()
+            if not last_transactions.exists():
+                continue
+
+            table = HistoryTable(last_transactions)
+            incoming = sum(tr.total for tr in last_transactions if tr.destination.pk == note.pk if tr.valid)
+            outcoming = sum(tr.total for tr in last_transactions if tr.source.pk == note.pk if tr.valid)
+            context = dict(
+                user=note.user,
+                table=table,
+                last_transactions=last_transactions,
+                incoming=incoming,
+                outcoming=outcoming,
+                diff=incoming - outcoming,
+                now=now,
+                last_report=last_report,
+            )
+            plain = render_to_string("note/mails/weekly_report.txt", context)
+            html = render_to_string("note/mails/weekly_report.html", context)
+            if options["debug"]:
+                self.stdout.write(plain)
+            else:
+                note.user.email_user("[Note Kfet] Rapport de la Note Kfet", plain, html_message=html)

management/commands/syncsql.py

@@ -0,0 +1,47 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from django.apps import apps
+from django.core.management.base import BaseCommand
+from django.db import connection
+from polymorphic.models import PolymorphicModel
+
+NO_SEQ = [
+    "Session",
+    "Token",
+    "WEIRole",  # dirty fix
+]
+
+
+class Command(BaseCommand):
+    """
+    Command to synchronise primary sequence of postgres after bulk insert of django.
+    """
+
+    def add_arguments(self,parser):
+        parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
+        return parser
+
+    def handle(self, *args, **kwargs):
+        app_list = kwargs["apps"]
+        if len(app_list):
+            model_classes = list()
+            for app in app_list:
+                model_classes += apps.get_app_config(app).get_models()
+        else:
+            # no app specified, sync everything
+            model_classes = apps.get_models(include_auto_created=True)
+
+        db_names = [
+            m._meta.db_table for m in model_classes
+            if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count() > 1
+        ]
+        com = "BEGIN;\n"
+        for db_name in db_names:
+            com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1),' \
+                   f' max("id") IS NOT null) FROM "{db_name}";\n'
+        com += "COMMIT;"
+        print(com)
+        cur = connection.cursor()
+        cur.execute(com)
+        cur.close()

shell/backup_db

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Create temporary backups directory
+mkdir -p /tmp/note-backups
+date=$(date +%Y-%m-%d)
+# Backup database and save it as tar archive
+sudo -u postgres pg_dump -F t note_db > "/tmp/note-backups/$date.sql"
+# Compress backup as gzip
+gzip "/tmp/note-backups/$date.sql"
+scp "/tmp/note-backups/$date.sql.gz" "club-bde@zamok.crans.org:backup/$date.sql.gz"

shell/docker_bash

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [ -r Dockerfile ]; then
+    if [ -w /var/run/docker.sock ]; then
+        docker build -t nk20 .
+        docker run -it -u $(id -u):$(id -g) --rm -v "$(pwd):/var/www/note_kfet/" -p 80:8080 nk20 bash
+    else
+        echo "Merci de rejoindre le groupe docker (ou lancez ce script en sudo) afin de pouvoir vous connecter au socket Docker."
+    fi
+else
+    echo "N'exécutez ce fichier que dans la racine de votre projet, afin de pouvoir localiser le fichier Dockerfile."
+fi

shell/tabularasa

@@ -1,7 +1,7 @@
 #!/usr/bin/sh
+sudo service postgresql stop
+sudo service postgresql start
 sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
 echo 'reset db';
-find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete
-./manage.py makemigrations
 ./manage.py migrate
 ./manage.py loaddata initial

templates/scripts/deleted_aliases.txt

@@ -0,0 +1,27 @@
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Ce message vous est envoyé automatiquement par la Note Kfet du BDE de
+l'ENS Cachan, à laquelle vous êtes inscrit·e. Si vous n'êtes plus
+adhérent·e, vous n'êtes pas nécessairement concerné·e par la suite
+de ce message.
+
+La Note Kfet 2020 vient d'être déployée, succédant à la Note Kfet 2015.
+Les données ont été migrées.
+
+Toutefois, la nouvelle note utilise un algorithme de normalisation des alias
+permettant de rechercher plus facilement un nom de note, et empêchant la
+création d'un alias trop proche d'un autre.
+
+Nous vous informons que les alias suivants ont été supprimés de votre compte,
+jugés trop proches d'autres alias déjà existants :
+
+{{ aliases_list|join:", " }}
+
+Nous nous excusons pour le désagrément, et espérons que vous pourrez
+profiter de la nouvelle Note Kfet.
+
+Cordialement,
+
+--
+Le BDE
+

templates/scripts/food_report.html

@@ -0,0 +1,51 @@
+{% load i18n %}
+{% now "Y-m-d" as today %}
+<!DOCTYPE html>
+<html lang="fr">
+<head>
+    <meta charset="UTF-8">
+    <title>[Note Kfet] Liste de la bouffe</title>
+</head>
+<body>
+<table>
+<thead>
+<tr>
+    <th>Club</th>
+    <th>Nom</th>
+    <th>Date de péremption</th>
+    <th>DLC/DDM</th>
+    <th>Consigne pour les GCKs</th>
+</tr>
+</thead>
+<tbody>
+    {% for food in foods %}
+    {% if today > food.expiry_date|date:"Y-m-d" %}
+	    {% if food.date_type and food.date_type == "DLC" %}
+		<tr bgcolor="red">
+	    {% else %}
+	        <tr bgcolor="yellow">
+	    {% endif %}
+	{% else %}
+	    <tr>
+	{% endif %}
+            <td>{{ food.owner.name }}</td>
+            <td>{{ food.name }}</td>
+            <td>{{ food.expiry_date }}</td>
+	    {% if food.date_type %}
+	        <td>{{ food.date_type }}</td>
+	    {% else %}
+            	<td>--</td>
+	    {% endif %}
+            <td>{{ food.order }}</td>
+        </tr>
+    {% endfor %}
+</tbody>
+</table>
+
+--
+<p>
+    Les GCKs du BDE<br>
+    {% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}
+</p>
+</body>
+</html>

templates/scripts/food_report.txt

@@ -0,0 +1,14 @@
+{% load i18n %}
+
+   Propriétaire   |         Nom         |  Date de péremption  | DLC/DDM |        Consigne pour les GCKs        |
+------------------+---------------------+----------------------+---------+---------------------------------------
+{% for food in foods %}
+
+{{ food.owner.name }} | {{ food.name }} | {{ food.expiry_date }} | {% if food.date_type %}{{ food.date_type }}{% else %} -- {% endif %} | {{ food.order }}
+
+{% endfor %}
+
+--
+Les GCKs du BDE
+
+{% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}

templates/scripts/horaires.html

@@ -0,0 +1,38 @@
+{% load getenv %}
+{% load i18n %}
+
+<!DOCTYPE html>
+<html lang="fr">
+<head>
+	<meta charset="UTF-8">
+	<title>Horaire du vote : {{ election_name}}</title>
+</head>
+<body>
+<p>
+	Bonjour {{ user.first_name }} {{ user.last_name }},
+</p>
+
+<p>
+	Nous t'informons que le vote : {{ election_name }}, sera ouvert de {{ time_start }} jusqu'à
+	{{ time_end }}.
+</p>
+
+<p>
+Tu peux voter autant de fois que tu le souhaites tant que le vote est ouvert.
+</p>
+
+<p>
+Le vote se déroulera sur la plateforme Belenios accessible via ce lien : <a href="{{ lien }}">{{ lien }}</a>
+</p>
+
+<p>
+Ce vote est organisé par l'Amicale des Élèves de l'École Normale Supérieure Paris-Saclay.
+</p>
+
+<p>
+En espérant que tu exerceras ton droit,<br>
+Le BDE<br>
+{% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}
+</p>
+</body>
+</html>

templates/scripts/horaires.txt

@@ -0,0 +1,17 @@
+{% load getenv %}
+{% load i18n %}
+
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Nous t'informons que le vote : {{ election_name }}, sera ouvert de {{ time_start }} jusqu'à {{ time_end }}.
+
+Tu peux voter autant de fois que tu le souhaites tant que le vote est ouvert.
+
+Le vote se déroulera sur la plateforme Belenios accessible via ce lien : {{ lien }}
+
+Ce vote est organisé par l'Amicale des Élèves de l'École Normale Supérieure Paris-Saclay.
+
+En espérant que tu exerceras ton droit,
+Le BDE
+
+{% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}

templates/scripts/intro_mail.html

@@ -0,0 +1,52 @@
+{% load getenv %}
+{% load i18n %}
+
+<!DOCTYPE html>
+<html lang="fr">
+<head>
+	<meta charset="UTF-8">
+	<title>Information : {{ election_name }})</title>
+</head>
+<body>
+<p>
+	Bonjour {{ user.first_name }} {{ user.last_name }},
+</p>
+
+<p>
+	Ce mail t'est envoyé car tu es inscrit·e sur la liste électorale pour le vote suivant : {{ election_name }}
+</p>
+
+<p>
+	Le vote se déroulera sur la plateforme Belenios accessible via ce lien : 
+	<a href="{{ lien }}">{{ lien }}</a>
+</p>
+
+<p>
+	Voici ton code d'électeur·ice pour pouvoir voter : {{ code_electeur }}
+</p>
+
+<p>
+	Une authentification par la Note Kfet (avec ta note : {{ user.username }}) sera nécessaire à la 	fin du vote pour le valider, si tu rencontres des problèmes pour réinitialiser ton mot de passe 	en cas d'oubli, n'hésites pas à envoyer un mail à
+	<a href="mailto:respo-info.bde@lists.crans.org">respo-info.bde@lists.crans.org</a>.
+</p>
+
+<p>
+Ce vote est organisé par l'Amicale des Élèves de l'École Normale Supérieure Paris-Saclay.
+</p>
+
+<p>
+Les personnes possédant une partie de la clé de déchiffrement sont :
+	<ul>
+	{% for a in autority %}
+	<li>{{ a }}</li>
+	{% endfor %}
+	</ul>
+</p>
+
+<p>
+En espérant que tu exerceras ce droit,<br>
+Le BDE<br>
+{% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}
+</p>
+</body>
+</html>

templates/scripts/intro_mail.txt

@@ -0,0 +1,25 @@
+{% load getenv %}
+{% load i18n %}
+
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Ce mail t'est envoyé car tu es inscrit·e sur la liste électorale pour le vote suivant : {{ election_name }}
+
+Le vote se déroulera sur la plateforme Belenios accessible via ce lien : {{ lien }}
+
+Voici ton code d'électeur·ice pour pouvoir voter : {{ code_electeur }}
+
+Une authentification par la Note Kfet (avec ta note : {{ user.username }}) sera nécessaire à la fin du vote pour le valider, si tu rencontres des problèmes pour réinitialiser ton mot de passe en cas d'oubli, n'hésites pas à envoyer un mail à respo-info.bde@lists.crans.org.
+
+Ce vote est organisé par l'Amicale des Élèves de l'École Normale Supérieure Paris-Saclay.
+
+Les personnes possédant une partie de la clé de déchiffrement sont :
+{% for a in autority %}
+{{ a }}
+{% endfor %}
+
+
+En espérant que tu exerceras ce droit,
+Le BDE
+
+{% trans "Mail generated by the Note Kfet on the" %} {% now "j F Y à H:i:s" %}

templates/scripts/unsupported_username.txt

@@ -0,0 +1,27 @@
+Bonjour {{ user.first_name }} {{ user.last_name }},
+
+Ce message vous est envoyé automatiquement par la Note Kfet du BDE de
+l'ENS Cachan, à laquelle vous êtes inscrit·e. Si vous n'êtes plus
+adhérent·e, vous n'êtes pas nécessairement concerné·e par la suite
+de ce message.
+
+La Note Kfet 2020 vient d'être déployée, succédant à la Note Kfet 2015.
+Les données ont été migrées.
+
+Toutefois, la nouvelle note utilise un algorithme de normalisation des alias
+permettant de rechercher plus facilement un nom de note, et empêchant la
+création d'un alias trop proche d'un autre.
+
+Nous vous informons que votre pseudo {{ old_username }} fait pas partie des
+alias problématiques. Il a été remplacé par le pseudo {{ new_username }},
+que vous devrez utiliser pour pouvoir vous connecter. Il sera ensuite
+possible de modifier votre pseudo.
+
+Nous nous excusons pour le désagrément, et espérons que vous pourrez
+profiter de la nouvelle Note Kfet.
+
+Cordialement,
+
+--
+Le BDE
+