package com.developx.szpitale.load; import com.developx.szpitale.model.*; import org.neo4j.driver.Driver; import org.neo4j.driver.Session; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; import java.util.stream.Collectors; public class GraphLoader { private static final Logger log = LoggerFactory.getLogger(GraphLoader.class); private static final int BATCH_SIZE = 1000; private final Driver driver; public GraphLoader(Driver driver) { this.driver = driver; } public LoadStats load(List datasets) { LoadStats stats = new LoadStats(); try (Session session = driver.session()) { for (CanonicalDataset dataset : datasets) { stats.addVoivodeship(dataset.voivodeship()); loadHospitals(session, dataset.hospitals(), stats); loadPeople(session, dataset.people(), stats); loadRoles(session, dataset.roles(), stats); loadAffiliations(session, dataset.affiliations(), stats); loadMandates(session, dataset.mandates(), stats); loadOrganizations(session, dataset.organizations(), stats); loadCompanies(session, dataset.companies(), stats); loadPersonOrganizationLinks(session, dataset.personOrganizationLinks(), stats); loadOrganizationLinks(session, dataset.organizationLinks(), stats); loadPersonCompanyLinks(session, dataset.personCompanyLinks(), stats); } } return stats; } private void loadHospitals(Session session, List hospitals, LoadStats stats) { List> batch = new ArrayList<>(); for (Hospital h : hospitals) { batch.add(Map.of( "id", h.id(), "name", h.name(), "shortName", h.shortName(), "city", h.city(), "voivodeship", h.voivodeship(), "legalForm", h.legalForm() != null ? h.legalForm().name() : null, "foundingBody", h.foundingBody(), "supervisoryBodyType", h.supervisoryBodyType() != null ? h.supervisoryBodyType().name() : null, "website", h.website(), "sourceUrls", h.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS h " + "MERGE (hospital:Hospital {id: h.id}) " + "SET hospital += {name:h.name, shortName:h.shortName, city:h.city, " + " voivodeship:h.voivodeship, legalForm:h.legalForm, " + " foundingBody:h.foundingBody, supervisoryBodyType:h.supervisoryBodyType, " + " website:h.website} " + "MERGE (v:Voivodeship {slug: h.voivodeship}) " + "MERGE (hospital)-[:W_WOJEWODZTWIE]->(v)"; session.run(cypher, Map.of("batch", subList)); stats.addHospitals(subList.size()); } } private void loadPeople(Session session, List people, LoadStats stats) { List> batch = new ArrayList<>(); for (Person p : people) { batch.add(Map.of( "id", p.id(), "fullName", p.fullName(), "displayName", p.displayName(), "titles", p.titles() != null ? p.titles() : Collections.emptyList(), "sourceUrls", p.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS p " + "MERGE (person:Person {id: p.id}) " + "SET person += {fullName:p.fullName, displayName:p.displayName, " + " titles:p.titles, sourceUrls:p.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addPeople(subList.size()); } } private void loadRoles(Session session, List roles, LoadStats stats) { Map> byRelType = new LinkedHashMap<>(); for (Role r : roles) { String relType = roleTypeToRelName(r); byRelType.computeIfAbsent(relType, k -> new ArrayList<>()).add(r); } for (Map.Entry> entry : byRelType.entrySet()) { String relType = entry.getKey(); List items = entry.getValue(); for (int i = 0; i < items.size(); i += BATCH_SIZE) { List subList = items.subList(i, Math.min(i + BATCH_SIZE, items.size())); String cypher = "UNWIND $batch AS r " + "MATCH (h:Hospital {id: r.hospitalId}) " + "MATCH (p:Person {id: r.personId}) " + "MERGE (h)-[rrel:`" + escapeBacktick(relType) + "`]->(p) " + "SET rrel += {roleType: r.roleType, roleLabel: r.roleLabel, " + " organ: r.organ, status: r.status, " + " validFrom: r.validFrom, validTo: r.validTo, note: r.note}"; session.run(cypher, Map.of("batch", subList.stream() .map(r -> Map.of( "hospitalId", r.hospitalId(), "personId", r.personId(), "roleType", r.roleType().name(), "roleLabel", r.roleLabel(), "organ", r.organ().name(), "status", r.status().name(), "validFrom", r.validFrom(), "validTo", r.validTo(), "note", r.note() )) .collect(Collectors.toList()))); stats.addRoles(subList.size()); } } } private void loadAffiliations(Session session, List affiliations, LoadStats stats) { for (int i = 0; i < affiliations.size(); i += BATCH_SIZE) { List subList = affiliations.subList( i, Math.min(i + BATCH_SIZE, affiliations.size())); String cypher = "UNWIND $batch AS a " + "MATCH (p:Person {id: a.personId}) " + "MERGE (party:Party {name: a.name}) SET party.type = a.type " + "MERGE (p)-[m:CZLONEK_PARTII]->(party) " + "SET m += {confidence: a.confidence, note: a.note, sourceUrls: a.sourceUrls}"; session.run(cypher, Map.of("batch", subList.stream() .map(a -> Map.of( "personId", a.personId(), "name", a.name(), "type", a.type().name(), "confidence", a.confidence().name(), "note", a.note(), "sourceUrls", a.sourceUrls() )) .collect(Collectors.toList()))); stats.addAffiliations(subList.size()); } } private void loadMandates(Session session, List mandates, LoadStats stats) { if (mandates.isEmpty()) return; for (int i = 0; i < mandates.size(); i += BATCH_SIZE) { List subList = mandates.subList( i, Math.min(i + BATCH_SIZE, mandates.size())); String cypher = "UNWIND $batch AS m " + "MATCH (p:Person {id: m.personId}) " + "MERGE (g:GovBody {name: m.body}) " + " SET g.type = m.mandateType, g.voivodeship = m.voivodeship, g.term = m.term " + "MERGE (p)-[:PELNI_MANDAT {mandateType: m.mandateType, term: m.term}]->(g)"; session.run(cypher, Map.of("batch", subList.stream() .map(m -> Map.of( "personId", m.personId(), "body", m.body(), "mandateType", m.mandateType().name(), "term", m.term(), "voivodeship", m.voivodeship(), "sourceUrls", m.sourceUrls() )) .collect(Collectors.toList()))); stats.addMandates(subList.size()); } } private void loadOrganizations(Session session, List organizations, LoadStats stats) { if (organizations.isEmpty()) return; List> batch = new ArrayList<>(); for (Organization o : organizations) { batch.add(Map.of( "id", o.id(), "name", o.name(), "krs", o.krs(), "nip", o.nip(), "sourceUrls", o.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS o " + "MERGE (org:Organization {id: o.id}) " + "SET org += {name: o.name, krs: o.krs, nip: o.nip, sourceUrls: o.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addOrganizations(subList.size()); } } private void loadCompanies(Session session, List companies, LoadStats stats) { if (companies.isEmpty()) return; List> batch = new ArrayList<>(); for (Company c : companies) { batch.add(Map.of( "id", c.id(), "name", c.name(), "krs", c.krs(), "nip", c.nip(), "ceidgId", c.ceidgId(), "sourceUrls", c.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS c " + "MERGE (comp:Company {id: c.id}) " + "SET comp += {name: c.name, krs: c.krs, nip: c.nip, ceidgId: c.ceidgId, sourceUrls: c.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addCompanies(subList.size()); } } private void loadPersonOrganizationLinks(Session session, List links, LoadStats stats) { if (links.isEmpty()) return; List> batch = new ArrayList<>(); for (PersonOrganizationLink l : links) { batch.add(Map.of( "personId", l.personId(), "organizationId", l.organizationId(), "roleLabel", l.roleLabel(), "basis", l.basis().name(), "sourceUrls", l.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS l " + "MATCH (p:Person {id: l.personId}), (org:Organization {id: l.organizationId}) " + "MERGE (p)-[m:CZLONEK_ORGANIZACJI]->(org) " + "SET m += {roleLabel: l.roleLabel, basis: l.basis, sourceUrls: l.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addPersonOrganizationLinks(subList.size()); } } private void loadOrganizationLinks(Session session, List links, LoadStats stats) { if (links.isEmpty()) return; List> batch = new ArrayList<>(); for (OrganizationLink l : links) { batch.add(Map.of( "fromId", l.fromOrganizationId(), "toId", l.toOrganizationId(), "basis", l.basis().name(), "note", l.note(), "sourceUrls", l.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS l " + "MATCH (a:Organization {id: l.fromId}), (b:Organization {id: l.toId}) " + "MERGE (a)-[m:POWIAZANA_Z]->(b) " + "SET m += {basis: l.basis, note: l.note, sourceUrls: l.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addOrganizationLinks(subList.size()); } } private void loadPersonCompanyLinks(Session session, List links, LoadStats stats) { if (links.isEmpty()) return; List> batch = new ArrayList<>(); for (PersonCompanyLink l : links) { batch.add(Map.of( "personId", l.personId(), "companyId", l.companyId(), "roleLabel", l.roleLabel(), "basis", l.basis().name(), "sourceUrls", l.sourceUrls() )); } for (int i = 0; i < batch.size(); i += BATCH_SIZE) { List> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size())); String cypher = "UNWIND $batch AS l " + "MATCH (p:Person {id: l.personId}), (comp:Company {id: l.companyId}) " + "MERGE (p)-[m:POWIAZANY_Z_FIRMA]->(comp) " + "SET m += {roleLabel: l.roleLabel, basis: l.basis, sourceUrls: l.sourceUrls}"; session.run(cypher, Map.of("batch", subList)); stats.addPersonCompanyLinks(subList.size()); } } private String roleTypeToRelName(Role r) { String organ = r.organ().name(); String role = r.roleType().name(); if ("DYREKCJA".equals(organ) && "DYREKTOR".equals(role)) return "DYREKTOR"; if ("DYREKCJA".equals(organ) && ("ZASTEPCA_DYREKTORA".equals(role) || "GLOWNY_KSIEGOWY".equals(role) || "PRZELOZONA_PIELEGNIARKOW".equals(role))) return "ZASTEPCA_DYREKTORA"; if ("PRZEWODNICZACY_ORGANU".equals(role)) return "PRZEWODNICZY_ORGANOWI"; return "CZLONEK_ORGANU"; } private String escapeBacktick(String s) { return s.replace("`", "\\`"); } public static class LoadStats { public int hospitalCount; public int personCount; public int roleCount; public int affiliationCount; public int mandateCount; public int organizationCount; public int companyCount; public int personOrganizationLinkCount; public int organizationLinkCount; public int personCompanyLinkCount; public final List voivodeships = new ArrayList<>(); public void addVoivodeship(String v) { voivodeships.add(v); } public void addHospitals(int count) { hospitalCount += count; } public void addPeople(int count) { personCount += count; } public void addRoles(int count) { roleCount += count; } public void addAffiliations(int count) { affiliationCount += count; } public void addMandates(int count) { mandateCount += count; } public void addOrganizations(int count) { organizationCount += count; } public void addCompanies(int count) { companyCount += count; } public void addPersonOrganizationLinks(int count) { personOrganizationLinkCount += count; } public void addOrganizationLinks(int count) { organizationLinkCount += count; } public void addPersonCompanyLinks(int count) { personCompanyLinkCount += count; } @Override public String toString() { return String.format( "Load stats - voivodeships: %s, hospitals: %d, people: %d, roles: %d, affiliations: %d, mandates: %d, organizations: %d, companies: %d, personOrgLinks: %d, orgLinks: %d, personCompanyLinks: %d", String.join(", ", voivodeships), hospitalCount, personCount, roleCount, affiliationCount, mandateCount, organizationCount, companyCount, personOrganizationLinkCount, organizationLinkCount, personCompanyLinkCount ); } } }