Mise à jour des AOMs 2025 (#4914)

Author: AntoineAugusti

apps/transport/lib/db/aom.ex

@@ -11,8 +11,6 @@ defmodule DB.AOM do
   alias Geo.MultiPolygon
 
   typed_schema "aom" do
-    # composition_res_id matches the id_reseau attribute from the Cerema dataset it’s the official ID of the AOM
-    field(:composition_res_id, :integer)
     field(:insee_commune_principale, :string)
     field(:siren, :string)
     field(:nom, :string)
@@ -31,6 +29,4 @@ defmodule DB.AOM do
   @spec get(insee_commune_principale: binary()) :: __MODULE__ | nil
   def get(insee_commune_principale: nil), do: nil
   def get(insee_commune_principale: insee), do: Repo.get_by(AOM, insee_commune_principale: insee)
-
-  def created_after_2021?(%__MODULE__{composition_res_id: composition_res_id}), do: composition_res_id >= 1_000
 end

apps/transport/lib/db/commune.ex

@@ -4,7 +4,7 @@ defmodule DB.Commune do
   """
   use Ecto.Schema
   use TypedEctoSchema
-  alias DB.{AOM, Departement, EPCI, Region}
+  alias DB.{Departement, EPCI, Region}
   alias Geo.MultiPolygon
 
   typed_schema "commune" do
@@ -15,9 +15,8 @@ defmodule DB.Commune do
     field(:population, :integer)
     field(:siren, :string)
     field(:arrondissement_insee, :string)
+    field(:aom_siren, :string)
 
-    # In theory a commune has only one AOM, the reference is done through the composition_res_id attribute
-    belongs_to(:aom_res, AOM, references: :composition_res_id)
     belongs_to(:region, Region)
     belongs_to(:departement, Departement, foreign_key: :departement_insee, references: :insee, type: :string)
     belongs_to(:epci, EPCI, foreign_key: :epci_insee, references: :insee, type: :string)

apps/transport/lib/mix/tasks/transport/import_aoms.ex

@@ -18,7 +18,7 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
 
   @shortdoc "Refreshes the database table `aom` with the latest data"
   use Mix.Task
-  import Ecto.{Query}
+  import Ecto.Query
   alias DB.{AOM, Commune, Region, Repo}
   require Logger
 
@@ -27,12 +27,12 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
   # download the Cerema file (.ods)
   # rename columns that are on two lines
   # export as CSV and publish as community resource
-  @aom_file "https://static.data.gouv.fr/resources/liste-et-composition-des-autorites-organisatrices-de-la-mobilite-aom/20241108-105224/liste-aoms-2024.csv"
+  @aom_file "https://www.data.gouv.fr/api/1/datasets/r/2e89a90a-6ca5-445b-b6c2-6f49cdeb8a2d"
   # Same for composition of each AOM, but no need even to rename columns
-  @aom_insee_file "https://static.data.gouv.fr/resources/liste-et-composition-des-autorites-organisatrices-de-la-mobilite-aom/20241122-154942/composition-communale-aom-2024.csv"
+  @aom_insee_file "https://www.data.gouv.fr/api/1/datasets/r/5195d58d-572e-4f13-8477-bbb11da42b93"
 
   # We don’t add collectivité d’outremer de Saint-Martin
-  @ignored_aom_ids ["312"]
+  @ignored_aom_sirens ["219711272"]
 
   @spec to_int(binary()) :: number() | nil
   def to_int(""), do: nil
@@ -58,13 +58,12 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
       Logger.info("aom #{nom} || previous region #{aom.region.nom} --- #{new_region.nom}")
     end
 
-    external_id = to_int(line["Id réseau"])
+    siren = line["N° SIREN"] |> String.trim()
 
-    {external_id,
+    {siren,
      Ecto.Changeset.change(aom, %{
-       composition_res_id: external_id,
        departement: extract_departement_insee(line["Département"]),
-       siren: line["N° SIREN"] |> String.trim(),
+       siren: siren,
        nom: nom,
        forme_juridique: normalize_forme(line["Forme juridique"]),
        # This is inconsistent with the real number of communes for some lines
@@ -91,6 +90,7 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
   defp normalize_forme("CC"), do: "Communauté de communes"
   defp normalize_forme("METRO"), do: "Métropole"
   defp normalize_forme("PETR"), do: "Pôle d'équilibre territorial et rural"
+  defp normalize_forme("POLEM"), do: "Pôle Métropolitain"
   defp normalize_forme(f), do: f
 
   @spec normalize_nom(binary()) :: binary()
@@ -109,11 +109,11 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
     old_aoms =
       AOM
       |> Repo.all()
-      |> Map.new(fn aom -> {aom.composition_res_id, aom} end)
+      |> Map.new(fn aom -> {aom.siren, aom} end)
 
     # get all the aom to import, outside of the transaction to reduce the time in the transaction
     # this already builds the changeset
-    # Mapset of {composition_res_id, changeset}
+    # Mapset of {aom_siren, changeset}
     aoms_to_add = get_aom_to_import() |> Enum.map(&changeset/1) |> MapSet.new()
 
     display_changes(old_aoms, aoms_to_add)
@@ -131,7 +131,7 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
           import_insee_aom()
           enable_trigger()
         end,
-        timeout: 1_000_000
+        timeout: :infinity
       )
 
     # we can then compute the aom geometries (the union of each cities geometries)
@@ -153,12 +153,12 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
     |> IO.binstream(:line)
     |> CSV.decode(separator: ?,, headers: true, validate_row_length: true)
     |> Enum.map(fn {:ok, line} -> line end)
-    |> Enum.reject(fn line -> line["Id réseau"] in (["", nil] ++ @ignored_aom_ids) end)
+    |> Enum.reject(fn line -> line["N° SIREN"] in (["", nil] ++ @ignored_aom_sirens) end)
   end
 
   defp existing_or_new_aom(line) do
     AOM
-    |> Repo.get_by(composition_res_id: to_int(line["Id réseau"]))
+    |> Repo.get_by(nom: normalize_nom(String.trim(line["Nom"])))
     |> case do
       nil ->
         %AOM{}
@@ -176,14 +176,14 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
   defp delete_old_aoms(aom_added, old_aoms) do
     Logger.info("deleting removed aom")
 
-    composition_res_id_added =
+    ids_added =
       aom_added
       |> Enum.map(fn {id, _changeset} -> id end)
       |> MapSet.new()
 
     old_aoms
-    |> Enum.each(fn {composition_res_id, old_aom} ->
-      unless MapSet.member?(composition_res_id_added, composition_res_id) do
+    |> Enum.each(fn {aom_siren, old_aom} ->
+      unless MapSet.member?(ids_added, aom_siren) do
         Logger.info("trying to delete old aom: #{old_aom.id} - #{old_aom.nom}")
 
         # Note: if the delete is impossible, you need to find what still depend on this aom,
@@ -194,7 +194,7 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
   end
 
   defp import_insee_aom do
-    Logger.info("Linking aoms to cities")
+    Logger.info("Linking AOMs to cities")
 
     {:ok, %HTTPoison.Response{status_code: 200, body: body}} =
       HTTPoison.get(@aom_insee_file, [], hackney: [follow_redirect: true])
@@ -204,15 +204,15 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
     stream
     |> IO.binstream(:line)
     |> CSV.decode(separator: ?,, headers: true, validate_row_length: true)
-    |> Enum.map(fn {:ok, line} -> {line["N° INSEE"], line["Id réseau"]} end)
-    |> Enum.reject(fn {_insee, id_reseau} -> id_reseau == "" || id_reseau == "-" end)
-    |> Enum.flat_map(fn {insee, id_reseau} ->
+    |> Enum.map(fn {:ok, line} -> {line["N° INSEE"], line["N° SIREN AOM"]} end)
+    |> Enum.reject(fn {_insee, siren_reseau} -> siren_reseau == "" || siren_reseau == "-" end)
+    |> Enum.flat_map(fn {insee, siren_reseau} ->
       # To reduce the number of UPDATE in the DB, we first check which city needs to be updated
       Commune
-      |> where([c], c.insee == ^insee and (c.aom_res_id != ^id_reseau or is_nil(c.aom_res_id)))
+      |> where([c], c.insee == ^insee and (c.aom_siren != ^siren_reseau or is_nil(c.aom_siren)))
       |> select([c], c.id)
       |> Repo.all()
-      |> Enum.map(fn c -> {c, id_reseau} end)
+      |> Enum.map(fn c -> {c, siren_reseau} end)
     end)
     |> Enum.reduce(%{}, fn {commune, aom}, commune_by_aom ->
       # Then we group those city by AO, to only do one UPDATE query for several cities
@@ -222,7 +222,7 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
     |> Enum.map(fn {aom, list_communes} ->
       Commune
       |> where([c], c.id in ^list_communes)
-      |> Repo.update_all(set: [aom_res_id: aom])
+      |> Repo.update_all(set: [aom_siren: aom])
     end)
   end
 
@@ -240,16 +240,16 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
                     SELECT
                     ST_UNION(commune.geom)
                     FROM commune
-                    WHERE commune.aom_res_id = ?
+                    WHERE commune.aom_siren = ?
                   )
                 """,
-                a.composition_res_id
+                a.siren
               )
           ]
         ]
       ),
       [],
-      timeout: 1_000_000
+      timeout: :infinity
     )
   end
 
@@ -258,54 +258,38 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
 
     max_for_each_aom =
       from(c in DB.Commune,
-        where: not is_nil(c.aom_res_id),
-        group_by: c.aom_res_id,
-        select: %{aom_res_id: c.aom_res_id, max_population: max(c.population)}
+        where: not is_nil(c.aom_siren),
+        group_by: c.aom_siren,
+        select: %{aom_siren: c.aom_siren, max_population: max(c.population)}
       )
 
     main_communes =
       from(c in DB.Commune,
-        where: not is_nil(c.aom_res_id),
+        where: not is_nil(c.aom_siren),
         join: max_for_each_aom in subquery(max_for_each_aom),
-        on: c.aom_res_id == max_for_each_aom.aom_res_id and c.population == max_for_each_aom.max_population,
-        select: [c.aom_res_id, c.insee]
+        on: c.aom_siren == max_for_each_aom.aom_siren and c.population == max_for_each_aom.max_population,
+        select: [c.aom_siren, c.insee]
       )
 
     main_communes =
       main_communes
       |> DB.Repo.all()
-      |> MapSet.new(fn [aom_res_id, insee] -> {aom_res_id, insee} end)
+      |> MapSet.new(fn [aom_siren, insee] -> {aom_siren, insee} end)
 
     {:ok, _} =
       Repo.transaction(
         fn ->
-          disable_trigger()
-
           main_communes
-          |> Enum.each(fn {aom_res_id, insee} ->
+          |> Enum.each(fn {aom_siren, insee} ->
             AOM
-            |> Repo.get_by!(composition_res_id: aom_res_id)
-            |> Ecto.Changeset.change(%{insee_commune_principale: insee})
-            |> Repo.update()
+            |> where([a], a.siren == ^aom_siren)
+            |> Repo.update_all(set: [insee_commune_principale: insee])
           end)
-
-          enable_trigger()
         end,
-        timeout: 1_000_000
+        timeout: :infinity
       )
   end
 
-  defp disable_trigger do
-    Repo.query!("ALTER TABLE aom DISABLE TRIGGER refresh_places_aom_trigger;")
-    Repo.query!("ALTER TABLE commune DISABLE TRIGGER refresh_places_commune_trigger;")
-  end
-
-  defp enable_trigger do
-    Repo.query!("ALTER TABLE aom ENABLE TRIGGER refresh_places_aom_trigger;")
-    Repo.query!("ALTER TABLE commune ENABLE TRIGGER refresh_places_commune_trigger;")
-    Repo.query!("REFRESH MATERIALIZED VIEW places;")
-  end
-
   defp migrate_datasets_to_new_aoms do
     queries = """
     -- This could be mostly automatized, you just have to look for a commune of the old AOM and see where it was migrated.
@@ -336,47 +320,71 @@ defmodule Mix.Tasks.Transport.ImportAOMs do
     --
     -- 2024
     -- Migrates a dataset to Pôle Métropolitain Mobilités Le Mans – Sarthe
-    update dataset_aom_legal_owner set aom_id = (select id from aom where composition_res_id = 1293) where aom_id IN (1283, 1285, 1288, 1292, 1294);
+    -- update dataset_aom_legal_owner set aom_id = (select id from aom where composition_res_id = 1293) where aom_id IN (1283, 1285, 1288, 1292, 1294);
+    -- 2025
+    -- Datasets still associated with deleted AOM as territory
+    -- %{
+    --   26 => [[26, "256900994", 1217], [26, "256900994", 871], [26, "256900994", 1216], [26, "256900994", 1259], [26, "256900994", 1139], [26, "256900994", 755], [26, "256900994", 695], [26, "256900994", 318]],
+    --   66 => [[66, "200011773", 1179], [66, "200011773", 164]],
+    --   168 => [[168, "246700967", 531]],
+    --   263 => [[263, "200016632", 940]],
+    --   1047 => [[1047, "243800984", 742]],
+    --   1301 => [[1301, "200070464", 1123]]
+    -- }
+    update dataset set aom_id = (select id from aom where siren = '200096386') where aom_id = 26;
+    update dataset set aom_id = (select id from aom where siren = '200075372') where aom_id = 66;
+    update dataset set aom_id = (select id from aom where siren = '200069680') where aom_id = 168;
+    update dataset set aom_id = (select id from aom where nom = 'Région Auvergne-Rhône-Alpes (CC du Bassin d''Aubenas)') where aom_id = 263;
+    update dataset set aom_id = (select id from aom where siren = '253800825') where aom_id = 1047;
+    update dataset set aom_id = (select id from aom where nom = 'Région Auvergne-Rhône-Alpes (CC Coeur de Maurienne Arvan)') where aom_id = 1301;
     """
 
     queries |> String.split(";") |> Enum.each(&Repo.query!/1)
   end
 
+  defp disable_trigger do
+    DB.Repo.query!("ALTER TABLE aom DISABLE TRIGGER aom_update_trigger;")
+  end
+
+  defp enable_trigger do
+    DB.Repo.query!("ALTER TABLE aom ENABLE TRIGGER aom_update_trigger;")
+  end
+
   defp display_changes(old_aoms, aoms_to_add) do
     mapset_first_elem_diff = fn a, b ->
       a |> MapSet.new(&elem(&1, 0)) |> MapSet.difference(b |> MapSet.new(&elem(&1, 0)))
     end
 
     new_aoms = mapset_first_elem_diff.(aoms_to_add, old_aoms)
     removed_aoms = mapset_first_elem_diff.(old_aoms, aoms_to_add)
-    Logger.info("#{new_aoms |> Enum.count()} new AOMs. reseau_id codes: #{Enum.join(new_aoms, ", ")}")
-    Logger.info("#{removed_aoms |> Enum.count()} removed AOMs. reseau_id codes: #{Enum.join(removed_aoms, ", ")}")
+    Logger.info("#{new_aoms |> Enum.count()} new AOMs. SIRENs: #{Enum.join(new_aoms, ", ")}")
+    Logger.info("#{removed_aoms |> Enum.count()} removed AOMs. SIRENs: #{Enum.join(removed_aoms, ", ")}")
 
     # Some Ecto fun: two ways of joining through assoc, see https://hexdocs.pm/ecto/associations.html
     deleted_aom_datasets =
       DB.Dataset
       |> join(:left, [d], aom in assoc(d, :aom))
-      |> where([d, aom], aom.composition_res_id in ^(removed_aoms |> MapSet.to_list()))
-      |> select([d, aom], [aom.id, aom.composition_res_id, d.id])
+      |> where([d, aom], aom.siren in ^(removed_aoms |> MapSet.to_list()))
+      |> select([d, aom], [aom.id, aom.siren, d.id])
       |> DB.Repo.all()
       |> Enum.group_by(&hd(&1))
 
     Logger.info(
-      "Datasets still associated with deleted AOM as territory (aom.id => [aom.id, aom.composition_res_id, dataset.id]) : #{inspect(deleted_aom_datasets)}"
+      "Datasets still associated with deleted AOM as territory (aom.id => [aom.id, aom.siren, dataset.id]) : #{inspect(deleted_aom_datasets)}"
     )
 
     deleted_legal_owners_query =
       from(d in DB.Dataset,
         # This magically works with the many_to_many
         join: aom in assoc(d, :legal_owners_aom),
-        where: aom.composition_res_id in ^(removed_aoms |> MapSet.to_list()),
-        select: [aom.id, aom.composition_res_id, d.id]
+        where: aom.siren in ^(removed_aoms |> MapSet.to_list()),
+        select: [aom.id, aom.siren, d.id]
       )
 
     deleted_legal_owners = deleted_legal_owners_query |> DB.Repo.all() |> Enum.group_by(&hd(&1))
 
     Logger.info(
-      "Datasets still associated with deleted AOM as legal owner (aom.id => [aom.id, aom.composition_res_id, dataset.id]): #{inspect(deleted_legal_owners)}"
+      "Datasets still associated with deleted AOM as legal owner (aom.id => [aom.id, aom.siren, dataset.id]): #{inspect(deleted_legal_owners)}"
     )
   end
 end