T1: Deduplicator compile fix + unit tests [checkpoint]
This commit is contained in:
Artur Kruszewski
@@ -0,0 +1,12 @@
|
||||
package com.developx.szpitale;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class SzpitaleGraphApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(SzpitaleGraphApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package com.developx.szpitale.ingest;
|
||||
|
||||
import com.developx.szpitale.model.*;
|
||||
import com.developx.szpitale.model.enums.*;
|
||||
import com.developx.szpitale.ingest.normalize.NameNormalizer;
|
||||
import com.developx.szpitale.ingest.normalize.PartyNormalizer;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.SerializationFeature;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class CanonicalWriter {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final NameNormalizer nameNormalizer = new NameNormalizer();
|
||||
private final PartyNormalizer partyNormalizer = new PartyNormalizer();
|
||||
|
||||
public CanonicalWriter() {
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
|
||||
}
|
||||
|
||||
public void write(Path outputDir, List<RawHospitalRecord> rawRecords, String voivodeship,
|
||||
IngestResults results) throws IOException {
|
||||
Path canonicalDir = outputDir.resolve("canonical").normalize();
|
||||
Files.createDirectories(canonicalDir);
|
||||
|
||||
List<Hospital> hospitals = new ArrayList<>();
|
||||
List<Person> people = new ArrayList<>();
|
||||
List<Role> roles = new ArrayList<>();
|
||||
List<Affiliation> affiliations = new ArrayList<>();
|
||||
|
||||
for (RawHospitalRecord rawRecord : rawRecords) {
|
||||
String hospitalId = nameNormalizer.makeHospitalId(voivodeship, rawRecord.hospitalName());
|
||||
|
||||
hospitals.add(new Hospital(
|
||||
hospitalId,
|
||||
rawRecord.hospitalName(),
|
||||
rawRecord.shortName(),
|
||||
rawRecord.city(),
|
||||
voivodeship,
|
||||
parseLegalForm(rawRecord.legalForm()),
|
||||
rawRecord.foundingBody(),
|
||||
parseSupervisoryBodyType(rawRecord.supervisoryBodyType()),
|
||||
rawRecord.nip(),
|
||||
rawRecord.krs(),
|
||||
rawRecord.website(),
|
||||
extractUniqueUrls(rawRecord.persons())
|
||||
));
|
||||
|
||||
Map<String, Person> peopleMap = new LinkedHashMap<>();
|
||||
Map<String, Set<String>> personUrlsMap = new LinkedHashMap<>();
|
||||
|
||||
for (RawHospitalRecord.RawPersonEntry personEntry : rawRecord.persons()) {
|
||||
String fullName = personEntry.fullName();
|
||||
if (fullName.isEmpty()) continue;
|
||||
if (fullName.toLowerCase().contains("brak") &&
|
||||
(personEntry.sourceUrl() == null || personEntry.sourceUrl().toLowerCase().contains("brak"))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
String personId = nameNormalizer.makePersonId(fullName);
|
||||
List<String> titles = nameNormalizer.extractTitles(fullName);
|
||||
String extractedName = nameNormalizer.extractFullName(fullName);
|
||||
|
||||
if (!peopleMap.containsKey(personId)) {
|
||||
peopleMap.put(personId, new Person(
|
||||
personId, extractedName, personEntry.displayTitle(),
|
||||
titles, new ArrayList<>()
|
||||
));
|
||||
personUrlsMap.put(personId, new LinkedHashSet<>());
|
||||
}
|
||||
|
||||
if (personEntry.sourceUrl() != null && !personEntry.sourceUrl().isEmpty()) {
|
||||
personUrlsMap.get(personId).add(personEntry.sourceUrl());
|
||||
}
|
||||
|
||||
RoleType roleType = parseRoleType(personEntry.function());
|
||||
OrganType organ = parseOrganType(personEntry.organ());
|
||||
RoleStatus status = parseStatus(personEntry.statusNote());
|
||||
|
||||
roles.add(new Role(
|
||||
hospitalId, personId, roleType,
|
||||
personEntry.function(),
|
||||
organ, status, null, null,
|
||||
null,
|
||||
List.of(personEntry.sourceUrl())
|
||||
));
|
||||
|
||||
String affText = personEntry.partyAffiliation();
|
||||
if (affText != null && !affText.isEmpty() &&
|
||||
!affText.toLowerCase().contains("brak") &&
|
||||
!affText.toLowerCase().contains("bezpartyj")) {
|
||||
|
||||
String canonicalParty = partyNormalizer.canonicalize(affText);
|
||||
AffiliationType affType = partyNormalizer.determineType(affText);
|
||||
ConfidenceLevel confidence = partyNormalizer.mapConfidence(personEntry.confidence());
|
||||
|
||||
String note = affText.contains("historycznie")
|
||||
? "historycznie: " + affText
|
||||
: affText;
|
||||
|
||||
affiliations.add(new Affiliation(
|
||||
personId, affType, canonicalParty, confidence,
|
||||
note,
|
||||
List.of(personEntry.sourceUrl())
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Update people with merged URLs
|
||||
List<Person> updatedPeople = new ArrayList<>();
|
||||
for (Map.Entry<String, Set<String>> entry : personUrlsMap.entrySet()) {
|
||||
Person p = peopleMap.get(entry.getKey());
|
||||
if (p != null) {
|
||||
updatedPeople.add(new Person(
|
||||
p.id(), p.fullName(), p.displayName(),
|
||||
p.titles(), new ArrayList<>(entry.getValue())
|
||||
));
|
||||
}
|
||||
}
|
||||
people.addAll(updatedPeople);
|
||||
}
|
||||
|
||||
CanonicalDataset dataset = new CanonicalDataset(
|
||||
"1.0", voivodeship, LocalDateTime.now(),
|
||||
hospitals, people, affiliations, roles, List.of()
|
||||
);
|
||||
|
||||
Path outputFile = canonicalDir.resolve(voivodeship + ".json");
|
||||
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), dataset);
|
||||
|
||||
results.addHospitals(hospitals.size());
|
||||
results.addPeople(people.size());
|
||||
results.addRoles(roles.size());
|
||||
results.addAffiliations(affiliations.size());
|
||||
}
|
||||
|
||||
private List<String> extractUniqueUrls(List<RawHospitalRecord.RawPersonEntry> persons) {
|
||||
return persons.stream()
|
||||
.map(RawHospitalRecord.RawPersonEntry::sourceUrl)
|
||||
.filter(url -> url != null && !url.isEmpty() && !url.toLowerCase().contains("brak"))
|
||||
.distinct()
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private LegalForm parseLegalForm(String form) {
|
||||
if (form == null) return LegalForm.SPZOZ;
|
||||
String upper = form.toUpperCase();
|
||||
if (upper.contains("SPOLKA")) return LegalForm.SPOLKA_Z_OO;
|
||||
if (upper.contains("PSYCHIATRYCZNY")) return LegalForm.SP_PSYCHIATRYCZNY_ZOZ;
|
||||
return LegalForm.SPZOZ;
|
||||
}
|
||||
|
||||
private SupervisoryBodyType parseSupervisoryBodyType(String type) {
|
||||
if (type == null) return SupervisoryBodyType.RADA_SPOLECZNA;
|
||||
return "NADZORCZA".equalsIgnoreCase(type)
|
||||
? SupervisoryBodyType.RADA_NADZORCZA
|
||||
: SupervisoryBodyType.RADA_SPOLECZNA;
|
||||
}
|
||||
|
||||
private RoleType parseRoleType(String function) {
|
||||
String lower = function.toLowerCase();
|
||||
if (lower.contains("dyrektor nacz")) {
|
||||
return RoleType.DYREKTOR;
|
||||
}
|
||||
if (lower.contains("dyrektor") && !lower.contains("z-ca") && !lower.contains("zast")) {
|
||||
return RoleType.DYREKTOR;
|
||||
}
|
||||
if (lower.contains("z-ca") || lower.contains("zast")) {
|
||||
return RoleType.ZASTEPCA_DYREKTORA;
|
||||
}
|
||||
if (lower.contains("glown") || lower.contains("gl. ")) {
|
||||
return RoleType.GLOWNY_KSIEGOWY;
|
||||
}
|
||||
if (lower.contains("naczel") && lower.contains("pieleg")) {
|
||||
return RoleType.PRZELOZONA_PIELEGNIARKOW;
|
||||
}
|
||||
if (lower.contains("przewodnicz") && !lower.contains("z-ca przewodnich")) {
|
||||
return RoleType.PRZEWODNICZACY_ORGANU;
|
||||
}
|
||||
if (lower.contains("sekte")) {
|
||||
return RoleType.SEKRETARZ_ORGANU;
|
||||
}
|
||||
if (lower.contains("z-ca")) {
|
||||
return RoleType.Z_CA_PRZEWODNICZACEGO;
|
||||
}
|
||||
return RoleType.CZLONEK_ORGANU_NADZORCZEGO;
|
||||
}
|
||||
|
||||
private OrganType parseOrganType(String organ) {
|
||||
if (organ == null) return OrganType.RADA_SPOLECZNA;
|
||||
String upper = organ.toUpperCase();
|
||||
if (upper.contains("DYREKCJA")) return OrganType.DYREKCJA;
|
||||
if (upper.contains("NADZORCZA")) return OrganType.RADA_NADZORCZA;
|
||||
return OrganType.RADA_SPOLECZNA;
|
||||
}
|
||||
|
||||
private RoleStatus parseStatus(String note) {
|
||||
if (note == null) return RoleStatus.AKTUALNY;
|
||||
String lower = note.toLowerCase();
|
||||
if (lower.contains("pelniac") || lower.contains("p.o.")) return RoleStatus.PELNIACY_OBOWIAZKI;
|
||||
if (lower.contains("elekt")) return RoleStatus.ELEKT;
|
||||
return RoleStatus.AKTUALNY;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.developx.szpitale.ingest;
|
||||
|
||||
import com.developx.szpitale.ingest.normalize.Deduplicator;
|
||||
import com.developx.szpitale.ingest.source.MarkdownRegistrySource;
|
||||
import com.developx.szpitale.ingest.source.VoivodeshipSource;
|
||||
import com.developx.szpitale.model.RawHospitalRecord;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class IngestCommand {
|
||||
|
||||
public String ingest(String voivodeship, String source, String outDir, boolean split) {
|
||||
if (!"markdown".equals(source)) {
|
||||
return "Error: Only 'markdown' source is currently supported.";
|
||||
}
|
||||
|
||||
Path outputDir = Path.of(outDir).toAbsolutePath().normalize();
|
||||
Path researchDir = Path.of("data").toAbsolutePath().normalize();
|
||||
|
||||
List<VoivodeshipSource> sources = MarkdownRegistrySource.discoverAllSources(researchDir);
|
||||
if (sources.isEmpty()) {
|
||||
return "Error: No research-*.{md,markdown} files found in data/ directory.";
|
||||
}
|
||||
|
||||
if (!"all".equals(voivodeship)) {
|
||||
String target = voivodeship.toLowerCase();
|
||||
sources = sources.stream()
|
||||
.filter(s -> s.voivodeship().equals(target))
|
||||
.collect(Collectors.toList());
|
||||
if (sources.isEmpty()) {
|
||||
return "Warning: No source found for voivodeship: " + voivodeship;
|
||||
}
|
||||
}
|
||||
|
||||
IngestResults totalResults = new IngestResults();
|
||||
CanonicalWriter writer = new CanonicalWriter();
|
||||
|
||||
for (VoivodeshipSource src : sources) {
|
||||
System.out.println("\nProcessing voivodeship: " + src.voivodeship());
|
||||
List<RawHospitalRecord> records = src.fetch();
|
||||
System.out.println(" Found " + records.size() + " hospital records");
|
||||
|
||||
IngestResults results = new IngestResults();
|
||||
try {
|
||||
writer.write(outputDir, records, src.voivodeship(), results);
|
||||
System.out.println(" Written: " + outputDir.resolve("canonical").resolve(src.voivodeship() + ".json"));
|
||||
} catch (IOException e) {
|
||||
System.err.println(" Error writing: " + e.getMessage());
|
||||
}
|
||||
totalResults.addHospitals(results.getHospitalCount());
|
||||
totalResults.addPeople(results.getPersonCount());
|
||||
totalResults.addRoles(results.getRoleCount());
|
||||
totalResults.addAffiliations(results.getAffiliationCount());
|
||||
}
|
||||
|
||||
try {
|
||||
writeIngestReport(outputDir, totalResults, sources);
|
||||
} catch (IOException e) {
|
||||
System.err.println("Error writing ingest report: " + e.getMessage());
|
||||
}
|
||||
|
||||
return totalResults.toString();
|
||||
}
|
||||
|
||||
private void writeIngestReport(Path outputDir, IngestResults results, List<VoivodeshipSource> sources) throws IOException {
|
||||
Path reportPath = outputDir.resolve("ingest-report.json");
|
||||
String report = "{" +
|
||||
"\"schemaVersion\":\"1.0\"," +
|
||||
"\"generatedAt\":\"" + java.time.LocalDateTime.now() + "\"," +
|
||||
"\"voivodeshipsProcessed\":" + sources.size() + "," +
|
||||
"\"hospitals\":" + results.getHospitalCount() + "," +
|
||||
"\"people\":" + results.getPersonCount() + "," +
|
||||
"\"roles\":" + results.getRoleCount() + "," +
|
||||
"\"affiliations\":" + results.getAffiliationCount() + "," +
|
||||
"\"gaps\":" + results.getGapCount() +
|
||||
"}";
|
||||
Files.writeString(reportPath, report);
|
||||
System.out.println("\nIngest report written to: " + reportPath);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.developx.szpitale.ingest;
|
||||
|
||||
public class IngestResults {
|
||||
private int hospitalCount = 0;
|
||||
private int personCount = 0;
|
||||
private int roleCount = 0;
|
||||
private int affiliationCount = 0;
|
||||
private int gapCount = 0;
|
||||
|
||||
public void addHospitals(int count) { hospitalCount += count; }
|
||||
public void addPeople(int count) { personCount += count; }
|
||||
public void addRoles(int count) { roleCount += count; }
|
||||
public void addAffiliations(int count) { affiliationCount += count; }
|
||||
public void addGaps(int count) { gapCount += count; }
|
||||
|
||||
public int getHospitalCount() { return hospitalCount; }
|
||||
public int getPersonCount() { return personCount; }
|
||||
public int getRoleCount() { return roleCount; }
|
||||
public int getAffiliationCount() { return affiliationCount; }
|
||||
public int getGapCount() { return gapCount; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format(
|
||||
"Ingest results: %d hospitals, %d people, %d roles, %d affiliations, %d gaps",
|
||||
hospitalCount, personCount, roleCount, affiliationCount, gapCount
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package com.developx.szpitale.ingest.normalize;
|
||||
|
||||
import com.developx.szpitale.model.Person;
|
||||
import org.apache.commons.text.similarity.JaroWinklerSimilarity;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class Deduplicator {
|
||||
|
||||
private static final double DUPLICATE_THRESHOLD = 0.85;
|
||||
private static final double SUSPECT_THRESHOLD = 0.70;
|
||||
|
||||
private final NameNormalizer nameNormalizer = new NameNormalizer();
|
||||
|
||||
public List<Person> deduplicate(List<Person> people) {
|
||||
List<String> personIds = new ArrayList<>();
|
||||
Map<String, List<Person>> bySlug = people.stream()
|
||||
.collect(Collectors.groupingBy(Person::id));
|
||||
|
||||
List<Person> result = new ArrayList<>();
|
||||
Set<String> processed = new HashSet<>();
|
||||
|
||||
for (Map.Entry<String, List<Person>> entry : bySlug.entrySet()) {
|
||||
List<Person> group = entry.getValue();
|
||||
if (group.size() == 1) {
|
||||
result.add(group.get(0));
|
||||
} else {
|
||||
// Same slug - merge source URLs
|
||||
Person merged = mergeBySlug(group);
|
||||
result.add(merged);
|
||||
}
|
||||
}
|
||||
|
||||
// Check cross-slug candidates with fuzzy matching
|
||||
checkFuzzyDuplicates(result);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private Person mergeBySlug(List<Person> duplicates) {
|
||||
Person first = duplicates.get(0);
|
||||
Set<String> allUrls = duplicates.stream()
|
||||
.flatMap(p -> p.sourceUrls().stream())
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
return new Person(
|
||||
first.id(),
|
||||
first.fullName(),
|
||||
first.displayName(),
|
||||
first.titles(),
|
||||
new ArrayList<>(allUrls)
|
||||
);
|
||||
}
|
||||
|
||||
private void checkFuzzyDuplicates(List<Person> people) {
|
||||
Map<String, List<String>> groups = new HashMap<>();
|
||||
for (int i = 0; i < people.size(); i++) {
|
||||
Person person = people.get(i);
|
||||
List<String> candidates = new ArrayList<>();
|
||||
for (int j = 0; j < i; j++) {
|
||||
Person existing = people.get(j);
|
||||
double similarity = nameNormalizer.fuzzySimilarity(
|
||||
person.fullName(),
|
||||
existing.fullName()
|
||||
);
|
||||
if (similarity >= SUSPECT_THRESHOLD && similarity < DUPLICATE_THRESHOLD) {
|
||||
candidates.add(existing.id());
|
||||
}
|
||||
}
|
||||
if (!candidates.isEmpty()) {
|
||||
groups.put(person.id(), candidates);
|
||||
}
|
||||
}
|
||||
|
||||
if (!groups.isEmpty()) {
|
||||
System.out.println("Warning: Possible name similarity conflicts (manual review recommended):");
|
||||
groups.forEach((id, candidates) ->
|
||||
System.out.println(" " + id + " -> similar to: " + String.join(", ", candidates)));
|
||||
}
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
package com.developx.szpitale.ingest.normalize;
|
||||
|
||||
import org.apache.commons.text.similarity.JaroWinklerSimilarity;
|
||||
|
||||
import java.text.Normalizer;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class NameNormalizer {
|
||||
|
||||
private static final Set<String> TITLE_TOKENS = Set.of(
|
||||
"prof.", "prof", "dr hab.", "dr.hab.", "dr.", "mgr", "mgr in\u017c.", "in\u017c.",
|
||||
"licencjat in\u017c.", "lek.", "lek.med.", "n. med.", "n. o zdr."
|
||||
);
|
||||
|
||||
private final JaroWinklerSimilarity jaroWinkler = new JaroWinklerSimilarity();
|
||||
|
||||
public String makeSlug(String fullName) {
|
||||
String normalized = Normalizer.normalize(fullName, Normalizer.Form.NFKD);
|
||||
normalized = normalized.replaceAll("\\p{M}", "");
|
||||
normalized = normalized.replaceAll("[^a-zA-Z0-9\\s-]", " ");
|
||||
normalized = normalized.toLowerCase().replaceAll("\\s+", "-").replaceAll("-+", "-");
|
||||
normalized = normalized.replaceAll("^-|-$", "");
|
||||
return normalized;
|
||||
}
|
||||
|
||||
public String extractFullName(String rawName) {
|
||||
return extractGivenAndFamilyNames(rawName);
|
||||
}
|
||||
|
||||
public List<String> extractTitles(String rawName) {
|
||||
List<String> titles = new ArrayList<>();
|
||||
String lower = rawName.toLowerCase();
|
||||
for (String title : TITLE_TOKENS) {
|
||||
if (lower.contains(title.toLowerCase())) {
|
||||
titles.add(title.trim());
|
||||
}
|
||||
}
|
||||
return titles;
|
||||
}
|
||||
|
||||
public String makePersonId(String fullName) {
|
||||
String slug = makeSlug(fullName);
|
||||
return "person:" + slug;
|
||||
}
|
||||
|
||||
public String makeHospitalId(String voivodeship, String hospitalName) {
|
||||
String slug = makeSlug(hospitalName);
|
||||
return "hosp:" + voivodeship + ":" + slug;
|
||||
}
|
||||
|
||||
public double fuzzySimilarity(String a, String b) {
|
||||
return jaroWinkler.apply(a.toLowerCase(), b.toLowerCase());
|
||||
}
|
||||
|
||||
public List<String> findDuplicateCandidates(List<String> names, String targetName, double threshold) {
|
||||
return names.stream()
|
||||
.filter(name -> fuzzySimilarity(name, targetName) >= threshold)
|
||||
.filter(name -> !name.equals(targetName))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private String extractGivenAndFamilyNames(String rawName) {
|
||||
Pattern pattern = Pattern.compile(
|
||||
"(?i)(prof\\.\\s*|dr\\s*hab\\.\\s*|dr\\.\\s*|mgr\\s*|in\u017c\\.\\s*)",
|
||||
Pattern.MULTILINE | Pattern.DOTALL
|
||||
);
|
||||
Matcher matcher = pattern.matcher(rawName);
|
||||
return matcher.replaceAll("").trim();
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
package com.developx.szpitale.ingest.normalize;
|
||||
|
||||
import com.developx.szpitale.model.enums.AffiliationType;
|
||||
import com.developx.szpitale.model.enums.ConfidenceLevel;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class PartyNormalizer {
|
||||
|
||||
private static final Map<String, String> PARTY_CANONICAL_MAPPINGS = Map.ofEntries(
|
||||
Map.entry("koalicja obywatelska", "Koalicja Obywatelska"),
|
||||
Map.entry("po", "Koalicja Obywatelska"),
|
||||
Map.entry("polska oferta", "Koalicja Obywatelska"),
|
||||
Map.entry("prawo i sprawiedliwosc", "Prawo i Sprawiedliwo\u015b\u0107"),
|
||||
Map.entry("pis", "Prawo i Sprawiedliwo\u015b\u0107"),
|
||||
Map.entry("psl", "PSL"),
|
||||
Map.entry("polska 2050", "Polska 2050"),
|
||||
Map.entry("trzecia droga", "Trzecia Droga"),
|
||||
Map.entry("polska 2050 pisl", "Trzecia Droga"),
|
||||
Map.entry("sl d", "SLD"),
|
||||
Map.entry("lewica", "Lewica"),
|
||||
Map.entry("prawica rzeczpospolitej", "Prawica Rzeczypospolitej"),
|
||||
Map.entry("ruch ludowy", "Ruch Ludowy"),
|
||||
Map.entry("kww", "KWW_LOKALNY")
|
||||
);
|
||||
|
||||
private static final Set<String> KOMMIT_SET = Set.of(
|
||||
"kww", "komitet wyborczy", "komitet lokalny", "komitet wyborc\u00f3w"
|
||||
);
|
||||
|
||||
public String canonicalize(String rawName) {
|
||||
String lower = rawName.toLowerCase().trim();
|
||||
if (PARTY_CANONICAL_MAPPINGS.containsKey(lower)) {
|
||||
return PARTY_CANONICAL_MAPPINGS.get(lower);
|
||||
}
|
||||
for (Map.Entry<String, String> entry : PARTY_CANONICAL_MAPPINGS.entrySet()) {
|
||||
if (lower.contains(entry.getKey())) {
|
||||
return entry.getValue();
|
||||
}
|
||||
}
|
||||
return rawName;
|
||||
}
|
||||
|
||||
public AffiliationType determineType(String rawName) {
|
||||
String lower = rawName.toLowerCase();
|
||||
if (lower.contains("kww") || lower.contains("komitet wyborc\u00f3w")) {
|
||||
return AffiliationType.KWW_LOKALNY;
|
||||
}
|
||||
if (lower.contains("komitet")) {
|
||||
return AffiliationType.KOMITET_WYBORCZY;
|
||||
}
|
||||
return AffiliationType.PARTIA;
|
||||
}
|
||||
|
||||
public ConfidenceLevel mapConfidence(String confidenceText) {
|
||||
if (confidenceText == null) return ConfidenceLevel.BRAK_DANYCH;
|
||||
String lower = confidenceText.toLowerCase();
|
||||
if (lower.contains("brak danych")) return ConfidenceLevel.BRAK_DANYCH;
|
||||
if (lower.contains("niezweryfik")) return ConfidenceLevel.NIEZWERYFIKOWANA;
|
||||
if (lower.contains("niepotwierdzona")) return ConfidenceLevel.NIEPOTWIERDZONA;
|
||||
return ConfidenceLevel.POTWIERDZONA;
|
||||
}
|
||||
}
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
package com.developx.szpitale.ingest.source;
|
||||
|
||||
import com.developx.szpitale.model.RawHospitalRecord;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class MarkdownRegistrySource implements VoivodeshipSource {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MarkdownRegistrySource.class);
|
||||
|
||||
private final Path researchDir;
|
||||
private final String voivodeshipSlug;
|
||||
|
||||
public MarkdownRegistrySource(Path researchDir, String voivodeshipSlug) {
|
||||
this.researchDir = researchDir;
|
||||
this.voivodeshipSlug = voivodeshipSlug;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String voivodeship() {
|
||||
return voivodeshipSlug;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RawHospitalRecord> fetch() {
|
||||
Path researchFile = findResearchFile(voivodeshipSlug);
|
||||
if (researchFile == null) {
|
||||
log.warn("No research file found for voivodeship: {}", voivodeshipSlug);
|
||||
return List.of();
|
||||
}
|
||||
return parseMarkdownFile(researchFile);
|
||||
}
|
||||
|
||||
private Path findResearchFile(String voivodeship) {
|
||||
Path path = researchDir.resolve("research-" + voivodeship + ".md");
|
||||
if (Files.exists(path)) {
|
||||
return path;
|
||||
}
|
||||
path = researchDir.resolve("research-" + voivodeship + ".markdown");
|
||||
if (Files.exists(path)) {
|
||||
return path;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<RawHospitalRecord> parseMarkdownFile(Path filePath) {
|
||||
List<RawHospitalRecord> records = new ArrayList<>();
|
||||
try {
|
||||
String content = Files.readString(filePath);
|
||||
parseContent(content, records);
|
||||
} catch (IOException e) {
|
||||
log.error("Failed to read research file: {}", filePath, e);
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
private void parseContent(String content, List<RawHospitalRecord> records) {
|
||||
String[] sections = content.split("## \\d+\\. ");
|
||||
for (int i = 1; i < sections.length; i++) {
|
||||
RawHospitalRecord record = parseHospitalSection(sections[i]);
|
||||
if (record != null) {
|
||||
records.add(record);
|
||||
}
|
||||
}
|
||||
// Also try to parse with just "## " prefix (fallback)
|
||||
if (records.isEmpty()) {
|
||||
String[] fallbackSections = content.split("## ");
|
||||
for (int i = 1; i < fallbackSections.length; i++) {
|
||||
if (!fallbackSections[i].contains("Nota metodologiczna")
|
||||
&& !fallbackSections[i].contains("Rozbiezności")
|
||||
&& !fallbackSections[i].contains("Główne luki")
|
||||
&& !fallbackSections[i].contains("Rekomendacje")) {
|
||||
RawHospitalRecord record = parseHospitalSection(fallbackSections[i]);
|
||||
if (record != null && record.hospitalName() != null && !record.hospitalName().contains("Nota")) {
|
||||
records.add(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RawHospitalRecord parseHospitalSection(String section) {
|
||||
String[] lines = section.split("\n", 3);
|
||||
if (lines.length < 2) return null;
|
||||
|
||||
String headerLine = lines[0].trim();
|
||||
String hospitalName = headerLine.replace("*", "").trim();
|
||||
|
||||
if (hospitalName.isEmpty() || hospitalName.startsWith("|")) return null;
|
||||
|
||||
String body = lines.length > 2 ? lines[2] : "";
|
||||
String legalForm = extractLegalForm(body);
|
||||
String foundingBody = extractFoundingBody(body);
|
||||
String supervisoryBodyType = extractSupervisoryBodyType(body);
|
||||
|
||||
List<RawHospitalRecord.RawPersonEntry> persons = parsePersonTables(section, legalForm, supervisoryBodyType,
|
||||
extractWebsite(section), hospitalName);
|
||||
|
||||
boolean hasData = persons.stream().anyMatch(p -> p.sourceUrl() != null && !p.sourceUrl().isEmpty() && !p.sourceUrl().contains("brak"));
|
||||
|
||||
return new RawHospitalRecord(
|
||||
hospitalName,
|
||||
extractShortName(hospitalName),
|
||||
extractCity(hospitalName),
|
||||
voivodeship(),
|
||||
legalForm,
|
||||
foundingBody,
|
||||
supervisoryBodyType,
|
||||
null,
|
||||
extractKrs(body),
|
||||
extractWebsite(headerLine + "\n" + body),
|
||||
persons
|
||||
);
|
||||
}
|
||||
|
||||
private List<RawHospitalRecord.RawPersonEntry> parsePersonTables(String section, String defaultLegalForm,
|
||||
String defaultSupervisory, String defaultWebsite, String hospitalName) {
|
||||
List<RawHospitalRecord.RawPersonEntry> entries = new ArrayList<>();
|
||||
|
||||
Document doc = Jsoup.parse(section);
|
||||
Elements tables = doc.select("table");
|
||||
|
||||
for (Element table : tables) {
|
||||
Elements rows = table.select("tr");
|
||||
if (rows.size() < 2) continue;
|
||||
|
||||
Elements headerCells = rows.get(0).select("th, td");
|
||||
boolean isPersonTable = headerCells.stream().anyMatch(h ->
|
||||
h.text().contains("Funkcja") && h.text().contains("Imię"));
|
||||
|
||||
if (!isPersonTable) continue;
|
||||
|
||||
for (int i = 1; i < rows.size(); i++) {
|
||||
Element row = rows.get(i);
|
||||
Elements cells = row.select("td");
|
||||
if (cells.isEmpty()) continue;
|
||||
|
||||
if (cells.get(0).text().contains("brak danych") ||
|
||||
cells.get(0).text().contains("brak imiennego")) {
|
||||
break;
|
||||
}
|
||||
|
||||
String function = cleanText(cells.get(0).html());
|
||||
String fullName = cleanText(cells.get(1).html());
|
||||
String affiliation = cells.size() > 2 ? cleanText(cells.get(2).html()) : "brak/niepotwierdzona";
|
||||
String sourceUrl = cells.size() > 3 ? extractUrl(cells.get(3).html()) : "";
|
||||
|
||||
if (fullName.isEmpty() || "brak".equalsIgnoreCase(fullName)) continue;
|
||||
if (function.contains("brak") && fullName.contains("brak danych")) continue;
|
||||
|
||||
String organ = detectOrgan(function, defaultSupervisory, defaultLegalForm);
|
||||
String confidence = mapConfidence(affiliation);
|
||||
String status = mapStatus(function);
|
||||
|
||||
// Extract display title from fullName
|
||||
String displayTitle = extractDisplayTitle(fullName, function);
|
||||
|
||||
entries.add(new RawHospitalRecord.RawPersonEntry(
|
||||
function, fullName, displayTitle, affiliation,
|
||||
confidence, sourceUrl, organ, status
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if (entries.isEmpty()) {
|
||||
// Fallback: try to parse single-line entries
|
||||
parseFallbackEntries(section, entries, defaultSupervisory, defaultLegalForm);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
private void parseFallbackEntries(String section, List<RawHospitalRecord.RawPersonEntry> entries,
|
||||
String defaultSupervisory, String defaultLegalForm) {
|
||||
String[] lines = section.split("\n");
|
||||
Pattern personPattern = Pattern.compile(
|
||||
"^\\s*\\|\\s*(Dyrektor|Z-ca|p\\.o\\.|Rada|Naczelna|Główne)\\b[^|]*\\|\\s*([^|]+)\\|([^|]*)\\|([^|]*)\\s*$",
|
||||
Pattern.MULTILINE
|
||||
);
|
||||
|
||||
for (String line : lines) {
|
||||
Matcher matcher = personPattern.matcher(line);
|
||||
if (matcher.find()) {
|
||||
String function = cleanText(matcher.group(1));
|
||||
String fullName = cleanText(matcher.group(2)).trim();
|
||||
String affiliation = cleanText(matcher.group(3)).trim();
|
||||
String sourceUrl = cleanText(matcher.group(4)).trim();
|
||||
|
||||
if (!fullName.isEmpty()) {
|
||||
entries.add(new RawHospitalRecord.RawPersonEntry(
|
||||
function, fullName, null, affiliation,
|
||||
mapConfidence(affiliation), sourceUrl,
|
||||
detectOrgan(function, defaultSupervisory, defaultLegalForm), ""
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String detectOrgan(String function, String defaultSupervisory, String defaultLegalForm) {
|
||||
String lower = function.toLowerCase();
|
||||
if (lower.contains("dyrektor") && !lower.contains("z-ca") && !lower.contains("p.o.")) {
|
||||
return "DYREKCJA";
|
||||
}
|
||||
if (lower.contains("z-ca") || lower.contains("p.o.") ||
|
||||
lower.contains("główn") || lower.contains("naczel") ||
|
||||
lower.contains("przel")) {
|
||||
return "DYREKCJA";
|
||||
}
|
||||
if (lower.contains("rada")) {
|
||||
if (defaultSupervisory != null && defaultSupervisory.contains("NADZORCZA")) {
|
||||
return "RADA_NADZORCZA";
|
||||
}
|
||||
return "RADA_SPOLECZNA";
|
||||
}
|
||||
if (defaultSupervisory != null && defaultSupervisory.contains("NADZORCZA")) {
|
||||
return "RADA_NADZORCZA";
|
||||
}
|
||||
return "RADA_SPOLECZNA";
|
||||
}
|
||||
|
||||
private String extractLegalForm(String body) {
|
||||
if (body.contains("sp. z o.o.") || body.contains("spółka")) {
|
||||
return "SPOLKA_Z_OO";
|
||||
}
|
||||
if (body.contains("Psychiatryczny")) {
|
||||
return "SP_PSYCHIATRYCZNY_ZOZ";
|
||||
}
|
||||
return "SPZOZ";
|
||||
}
|
||||
|
||||
private String extractFoundingBody(String body) {
|
||||
var mat = Pattern.compile("Organ tworzący:\\s*(.+?)\\.(?:\\s|$)", Pattern.MULTILINE | Pattern.DOTALL).matcher(body);
|
||||
if (mat.find()) {
|
||||
return cleanText(mat.group(1)).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractSupervisoryBodyType(String body) {
|
||||
if (body.contains("Rada Nadzorcza") || body.contains("RADA NADZORCZA")) {
|
||||
return "RADA_NADZORCZA";
|
||||
}
|
||||
return "RADA_SPOLECZNA";
|
||||
}
|
||||
|
||||
private String extractKrs(String body) {
|
||||
var mat = Pattern.compile("(?:KRS\\s*|krs\\s*:\\s*)(\\d{8,10})").matcher(body);
|
||||
if (mat.find()) {
|
||||
return mat.group(1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractWebsite(String text) {
|
||||
var mat = Pattern.compile("(https?://[^\\s|]+)").matcher(text);
|
||||
if (mat.find()) {
|
||||
return mat.group(1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractShortName(String fullName) {
|
||||
if (fullName.contains("Kliniczny") && fullName.contains("w ")) {
|
||||
var mat = Pattern.compile("([A-Z]{2,4})\\b")
|
||||
.matcher(fullName.split("Kliniczny")[0]);
|
||||
if (mat.find()) return "USK";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractCity(String hospitalName) {
|
||||
var mat = Pattern.compile("w\\s+([^,.(]+)").matcher(hospitalName);
|
||||
if (mat.find()) {
|
||||
return cleanText(mat.group(1)).trim();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String extractDisplayTitle(String fullName, String function) {
|
||||
if (function.contains("Dyrektor Naczelny")) {
|
||||
return "Dyrektor Naczelny";
|
||||
}
|
||||
if (function.contains("p.o.")) {
|
||||
return "p.o. " + function.replace("p.o. ", "");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String mapConfidence(String affiliation) {
|
||||
if (affiliation == null) return "BRAK_DANYCH";
|
||||
String lower = affiliation.toLowerCase();
|
||||
if (lower.contains("brak danych")) return "BRAK_DANYCH";
|
||||
if (lower.contains("niezweryfik") || lower.contains("NIEZWERYFIKOWANE")) return "NIEZWERYFIKOWANA";
|
||||
if (lower.contains("brak/niepotwierdzona") || lower.contains("niepotwierdzona") ||
|
||||
lower.contains("niepotwierdzone") || lower.contains("niepotwierdzony")) return "NIEPOTWIERDZONA";
|
||||
if (lower.contains("NIEZWERYFIKOWANE (zbieżność") || lower.contains("nieweryfikow")) return "NIEZWERYFIKOWANA";
|
||||
if (affiliation.contains("KO") || affiliation.contains("PO") ||
|
||||
affiliation.contains("PiS") || affiliation.contains("PSL") ||
|
||||
affiliation.contains("Trzecia Droga") || affiliation.contains("Polska 2050") ||
|
||||
affiliation.contains("KWW ") || affiliation.contains("KOMITET") ||
|
||||
affiliation.contains("Prawica") || affiliation.contains("SLD") ||
|
||||
affiliation.contains("Lewica")) {
|
||||
return "POTWIERDZONA";
|
||||
}
|
||||
return "NIEPOTWIERDZONA";
|
||||
}
|
||||
|
||||
private String mapStatus(String function) {
|
||||
String lower = function.toLowerCase();
|
||||
if (lower.contains("p.o.") || lower.contains("pełniący")) return "PELNIACY_OBOWIAZKI";
|
||||
if (lower.contains("elekt")) return "ELEKT";
|
||||
if (lower.contains("były") || lower.contains("dawniej")) return "BYLY";
|
||||
return "AKTUALNY";
|
||||
}
|
||||
|
||||
private String cleanText(String html) {
|
||||
if (html == null) return "";
|
||||
Document doc = Jsoup.parse(html);
|
||||
String text = doc.text().trim();
|
||||
// Remove trailing pipe if any
|
||||
if (text.endsWith("|")) text = text.substring(0, text.length() - 1).trim();
|
||||
return text;
|
||||
}
|
||||
|
||||
private String extractUrl(String text) {
|
||||
String cleaned = cleanText(text);
|
||||
var mat = Pattern.compile("(https?://[^\\s]+)").matcher(cleaned);
|
||||
if (mat.find()) return mat.group(1).replaceAll("[)]+$", "");
|
||||
return cleaned.isEmpty() ? "" : cleaned;
|
||||
}
|
||||
|
||||
public static List<VoivodeshipSource> discoverAllSources(Path researchDir) {
|
||||
List<VoivodeshipSource> sources = new ArrayList<>();
|
||||
try (var paths = Files.list(researchDir)) {
|
||||
paths.filter(p -> p.toString().endsWith(".md"))
|
||||
.filter(p -> p.getFileName().toString().startsWith("research-"))
|
||||
.forEach(p -> {
|
||||
String voiv = p.getFileName().toString().replace("research-", "").replace(".md", "");
|
||||
sources.add(new MarkdownRegistrySource(researchDir, voiv));
|
||||
log.info("Discovered research source for voivodeship: {}", voiv);
|
||||
});
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not scan research directory: {}", researchDir);
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.developx.szpitale.ingest.source;
|
||||
|
||||
import com.developx.szpitale.model.RawHospitalRecord;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface VoivodeshipSource {
|
||||
String voivodeship();
|
||||
List<RawHospitalRecord> fetch();
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.developx.szpitale.load;
|
||||
|
||||
import com.developx.szpitale.model.CanonicalDataset;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class CanonicalReader {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
public CanonicalReader() {
|
||||
objectMapper.registerModule(new JavaTimeModule());
|
||||
}
|
||||
|
||||
public List<CanonicalDataset> readAll(Path inputDir, String voivodeship) throws IOException {
|
||||
List<CanonicalDataset> datasets = new ArrayList<>();
|
||||
Path canonicalDir = inputDir.resolve("canonical").toAbsolutePath().normalize();
|
||||
|
||||
if (!Files.exists(canonicalDir)) {
|
||||
System.err.println("Error: canonical directory not found: " + canonicalDir);
|
||||
return datasets;
|
||||
}
|
||||
|
||||
try (var paths = Files.list(canonicalDir)) {
|
||||
paths.filter(p -> p.getFileName().toString().endsWith(".json"))
|
||||
.filter(p -> {
|
||||
if ("all".equalsIgnoreCase(voivodeship)) return true;
|
||||
String base = p.getFileName().toString().replace(".json", "");
|
||||
return base.equalsIgnoreCase(voivodeship);
|
||||
})
|
||||
.forEach(p -> {
|
||||
try {
|
||||
CanonicalDataset ds = objectMapper.readValue(p.toFile(), CanonicalDataset.class);
|
||||
datasets.add(ds);
|
||||
} catch (IOException e) {
|
||||
System.err.println("Error reading " + p + ": " + e.getMessage());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return datasets;
|
||||
}
|
||||
|
||||
public CanonicalDataset readSingle(Path inputDir, String voivodeship) throws IOException {
|
||||
return readAll(inputDir, voivodeship).stream().findFirst().orElse(null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package com.developx.szpitale.load;
|
||||
|
||||
import com.developx.szpitale.model.*;
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.Session;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class GraphLoader {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GraphLoader.class);
|
||||
private static final int BATCH_SIZE = 1000;
|
||||
|
||||
private final Driver driver;
|
||||
|
||||
public GraphLoader(Driver driver) {
|
||||
this.driver = driver;
|
||||
}
|
||||
|
||||
public LoadStats load(List<CanonicalDataset> datasets) {
|
||||
LoadStats stats = new LoadStats();
|
||||
try (Session session = driver.session()) {
|
||||
for (CanonicalDataset dataset : datasets) {
|
||||
stats.addVoivodeship(dataset.voivodeship());
|
||||
loadHospitals(session, dataset.hospitals(), stats);
|
||||
loadPeople(session, dataset.people(), stats);
|
||||
loadRoles(session, dataset.roles(), stats);
|
||||
loadAffiliations(session, dataset.affiliations(), stats);
|
||||
loadMandates(session, dataset.mandates(), stats);
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
||||
private void loadHospitals(Session session, List<Hospital> hospitals, LoadStats stats) {
|
||||
List<Map<String, Object>> batch = new ArrayList<>();
|
||||
for (Hospital h : hospitals) {
|
||||
batch.add(Map.of(
|
||||
"id", h.id(),
|
||||
"name", h.name(),
|
||||
"shortName", h.shortName(),
|
||||
"city", h.city(),
|
||||
"voivodeship", h.voivodeship(),
|
||||
"legalForm", h.legalForm() != null ? h.legalForm().name() : null,
|
||||
"foundingBody", h.foundingBody(),
|
||||
"supervisoryBodyType", h.supervisoryBodyType() != null ? h.supervisoryBodyType().name() : null,
|
||||
"website", h.website(),
|
||||
"sourceUrls", h.sourceUrls()
|
||||
));
|
||||
}
|
||||
|
||||
for (int i = 0; i < batch.size(); i += BATCH_SIZE) {
|
||||
List<Map<String, Object>> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size()));
|
||||
String cypher = "UNWIND $batch AS h " +
|
||||
"MERGE (hospital:Hospital {id: h.id}) " +
|
||||
"SET hospital += {name:h.name, shortName:h.shortName, city:h.city, " +
|
||||
" voivodeship:h.voivodeship, legalForm:h.legalForm, " +
|
||||
" foundingBody:h.foundingBody, supervisoryBodyType:h.supervisoryBodyType, " +
|
||||
" website:h.website} " +
|
||||
"MERGE (v:Voivodeship {slug: h.voivodeship}) " +
|
||||
"MERGE (hospital)-[:W_WOJEWODZTWIE]->(v)";
|
||||
session.run(cypher, Map.of("batch", subList));
|
||||
stats.addHospitals(subList.size());
|
||||
}
|
||||
}
|
||||
|
||||
private void loadPeople(Session session, List<Person> people, LoadStats stats) {
|
||||
List<Map<String, Object>> batch = new ArrayList<>();
|
||||
for (Person p : people) {
|
||||
batch.add(Map.of(
|
||||
"id", p.id(),
|
||||
"fullName", p.fullName(),
|
||||
"displayName", p.displayName(),
|
||||
"titles", p.titles() != null ? p.titles() : Collections.emptyList(),
|
||||
"sourceUrls", p.sourceUrls()
|
||||
));
|
||||
}
|
||||
|
||||
for (int i = 0; i < batch.size(); i += BATCH_SIZE) {
|
||||
List<Map<String, Object>> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size()));
|
||||
String cypher = "UNWIND $batch AS p " +
|
||||
"MERGE (person:Person {id: p.id}) " +
|
||||
"SET person += {fullName:p.fullName, displayName:p.displayName, " +
|
||||
" titles:p.titles, sourceUrls:p.sourceUrls}";
|
||||
session.run(cypher, Map.of("batch", subList));
|
||||
stats.addPeople(subList.size());
|
||||
}
|
||||
}
|
||||
|
||||
private void loadRoles(Session session, List<Role> roles, LoadStats stats) {
|
||||
Map<String, List<Role>> byRelType = new LinkedHashMap<>();
|
||||
for (Role r : roles) {
|
||||
String relType = roleTypeToRelName(r);
|
||||
byRelType.computeIfAbsent(relType, k -> new ArrayList<>()).add(r);
|
||||
}
|
||||
|
||||
for (Map.Entry<String, List<Role>> entry : byRelType.entrySet()) {
|
||||
String relType = entry.getKey();
|
||||
List<Role> items = entry.getValue();
|
||||
|
||||
for (int i = 0; i < items.size(); i += BATCH_SIZE) {
|
||||
List<Role> subList = items.subList(i, Math.min(i + BATCH_SIZE, items.size()));
|
||||
String cypher = "UNWIND $batch AS r " +
|
||||
"MATCH (h:Hospital {id: r.hospitalId}) " +
|
||||
"MATCH (p:Person {id: r.personId}) " +
|
||||
"MERGE (h)-[rrel:`" + escapeBacktick(relType) + "`]->(p) " +
|
||||
"SET rrel += {roleType: r.roleType, roleLabel: r.roleLabel, " +
|
||||
" organ: r.organ, status: r.status, " +
|
||||
" validFrom: r.validFrom, validTo: r.validTo, note: r.note}";
|
||||
session.run(cypher, Map.of("batch", subList.stream()
|
||||
.map(r -> Map.of(
|
||||
"hospitalId", r.hospitalId(),
|
||||
"personId", r.personId(),
|
||||
"roleType", r.roleType().name(),
|
||||
"roleLabel", r.roleLabel(),
|
||||
"organ", r.organ().name(),
|
||||
"status", r.status().name(),
|
||||
"validFrom", r.validFrom(),
|
||||
"validTo", r.validTo(),
|
||||
"note", r.note()
|
||||
))
|
||||
.collect(Collectors.toList())));
|
||||
stats.addRoles(subList.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void loadAffiliations(Session session, List<Affiliation> affiliations, LoadStats stats) {
|
||||
for (int i = 0; i < affiliations.size(); i += BATCH_SIZE) {
|
||||
List<Affiliation> subList = affiliations.subList(
|
||||
i, Math.min(i + BATCH_SIZE, affiliations.size()));
|
||||
String cypher = "UNWIND $batch AS a " +
|
||||
"MATCH (p:Person {id: a.personId}) " +
|
||||
"MERGE (party:Party {name: a.name}) SET party.type = a.type " +
|
||||
"MERGE (p)-[m:CZLONEK_PARTII]->(party) " +
|
||||
"SET m += {confidence: a.confidence, note: a.note, sourceUrls: a.sourceUrls}";
|
||||
session.run(cypher, Map.of("batch", subList.stream()
|
||||
.map(a -> Map.of(
|
||||
"personId", a.personId(),
|
||||
"name", a.name(),
|
||||
"type", a.type().name(),
|
||||
"confidence", a.confidence().name(),
|
||||
"note", a.note(),
|
||||
"sourceUrls", a.sourceUrls()
|
||||
))
|
||||
.collect(Collectors.toList())));
|
||||
stats.addAffiliations(subList.size());
|
||||
}
|
||||
}
|
||||
|
||||
private void loadMandates(Session session, List<Mandate> mandates, LoadStats stats) {
|
||||
if (mandates.isEmpty()) return;
|
||||
|
||||
for (int i = 0; i < mandates.size(); i += BATCH_SIZE) {
|
||||
List<Mandate> subList = mandates.subList(
|
||||
i, Math.min(i + BATCH_SIZE, mandates.size()));
|
||||
String cypher = "UNWIND $batch AS m " +
|
||||
"MATCH (p:Person {id: m.personId}) " +
|
||||
"MERGE (g:GovBody {name: m.body}) " +
|
||||
" SET g.type = m.mandateType, g.voivodeship = m.voivodeship, g.term = m.term " +
|
||||
"MERGE (p)-[:PELNI_MANDAT {mandateType: m.mandateType, term: m.term}]->(g)";
|
||||
session.run(cypher, Map.of("batch", subList.stream()
|
||||
.map(m -> Map.of(
|
||||
"personId", m.personId(),
|
||||
"body", m.body(),
|
||||
"mandateType", m.mandateType().name(),
|
||||
"term", m.term(),
|
||||
"voivodeship", m.voivodeship(),
|
||||
"sourceUrls", m.sourceUrls()
|
||||
))
|
||||
.collect(Collectors.toList())));
|
||||
stats.addMandates(subList.size());
|
||||
}
|
||||
}
|
||||
|
||||
private String roleTypeToRelName(Role r) {
|
||||
String organ = r.organ().name();
|
||||
String role = r.roleType().name();
|
||||
|
||||
if ("DYREKCJA".equals(organ) && "DYREKTOR".equals(role)) return "DYREKTOR";
|
||||
if ("DYREKCJA".equals(organ) && ("ZASTEPCA_DYREKTORA".equals(role) || "GLOWNY_KSIEGOWY".equals(role) || "PRZELOZONA_PIELEGNIARKOW".equals(role))) return "ZASTEPCA_DYREKTORA";
|
||||
if ("PRZEWODNICZACY_ORGANU".equals(role)) return "PRZEWODNICZY_ORGANOWI";
|
||||
return "CZLONEK_ORGANU";
|
||||
}
|
||||
|
||||
private String escapeBacktick(String s) {
|
||||
return s.replace("`", "\\`");
|
||||
}
|
||||
|
||||
public static class LoadStats {
|
||||
public int hospitalCount;
|
||||
public int personCount;
|
||||
public int roleCount;
|
||||
public int affiliationCount;
|
||||
public int mandateCount;
|
||||
public final List<String> voivodeships = new ArrayList<>();
|
||||
|
||||
public void addVoivodeship(String v) { voivodeships.add(v); }
|
||||
public void addHospitals(int count) { hospitalCount += count; }
|
||||
public void addPeople(int count) { personCount += count; }
|
||||
public void addRoles(int count) { roleCount += count; }
|
||||
public void addAffiliations(int count) { affiliationCount += count; }
|
||||
public void addMandates(int count) { mandateCount += count; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format(
|
||||
"Load stats - voivodeships: %s, hospitals: %d, people: %d, roles: %d, affiliations: %d, mandates: %d",
|
||||
String.join(", ", voivodeships), hospitalCount, personCount, roleCount, affiliationCount, mandateCount
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.developx.szpitale.load;
|
||||
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.Session;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class GraphSchema {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GraphSchema.class);
|
||||
|
||||
private final Driver driver;
|
||||
|
||||
public GraphSchema(Driver driver) {
|
||||
this.driver = driver;
|
||||
}
|
||||
|
||||
public void createConstraints() {
|
||||
try (Session session = driver.session()) {
|
||||
session.run("CREATE CONSTRAINT hospital_id IF NOT EXISTS FOR (h:Hospital) REQUIRE h.id IS UNIQUE");
|
||||
session.run("CREATE CONSTRAINT person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.id IS UNIQUE");
|
||||
session.run("CREATE CONSTRAINT party_name IF NOT EXISTS FOR (x:Party) REQUIRE x.name IS UNIQUE");
|
||||
session.run("CREATE CONSTRAINT voiv_slug IF NOT EXISTS FOR (v:Voivodeship) REQUIRE v.slug IS UNIQUE");
|
||||
session.run("CREATE INDEX person_name IF NOT EXISTS FOR (p:Person) ON (p.fullName)");
|
||||
session.run("CREATE INDEX govbody_name IF NOT EXISTS FOR (g:GovBody) ON (g.name)");
|
||||
log.info("Graph constraints and indexes created");
|
||||
}
|
||||
}
|
||||
|
||||
public void dropAll() {
|
||||
try (Session session = driver.session()) {
|
||||
session.run("MATCH (n) DETACH DELETE n");
|
||||
log.info("All nodes and relationships deleted");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package com.developx.szpitale.load;
|
||||
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.Session;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
|
||||
public class GraphValidator {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(GraphValidator.class);
|
||||
private final Driver driver;
|
||||
|
||||
public GraphValidator(Driver driver) {
|
||||
this.driver = driver;
|
||||
}
|
||||
|
||||
public ValidationReport validate() {
|
||||
ValidationReport report = new ValidationReport();
|
||||
try (Session session = driver.session()) {
|
||||
|
||||
// 1. Hospitals without directors
|
||||
var result1 = session.run(
|
||||
"MATCH (h:Hospital) WHERE NOT (h)-[:DYREKTOR]->() " +
|
||||
"RETURN h.name AS hospital, h.city AS city, h.voivodeship AS voivodeship " +
|
||||
"ORDER BY h.voivodeship, h.name");
|
||||
List<Map<String, Object>> hospitalsWithoutDirector = new ArrayList<>();
|
||||
result1.list().forEach(r -> hospitalsWithoutDirector.add(r.asMap()));
|
||||
report.setHospitalsWithoutDirector(hospitalsWithoutDirector);
|
||||
|
||||
// 2. People with mandates linking to hospital roles
|
||||
var result2 = session.run(
|
||||
"MATCH (h:Hospital)-[r:CZLONEK_ORGANU|PRZEWODNICZY_ORGANOWI]->(p:Person)-[:PELNI_MANDAT]->(g:GovBody) " +
|
||||
"RETURN p.fullName AS person, " +
|
||||
" collect(DISTINCT h.name) AS szpitale, " +
|
||||
" collect(DISTINCT g.name) AS mandaty, " +
|
||||
" collect(DISTINCT labels(p)) AS labels " +
|
||||
"ORDER BY person");
|
||||
List<Map<String, Object>> overlaps = new ArrayList<>();
|
||||
result2.list().forEach(r -> overlaps.add(r.asMap()));
|
||||
report.setOverlapReport(overlaps);
|
||||
|
||||
// 3. People in multiple hospitals
|
||||
var result3 = session.run(
|
||||
"MATCH (h:Hospital)-[r:CZLONEK_ORGANU|PRZEWODNICZY_ORGANOWI]->(p:Person) " +
|
||||
"WITH p, collect(DISTINCT h.name) AS szpitale WHERE size(szpitale) > 1 " +
|
||||
"RETURN p.fullName AS person, szpitale, count(p) AS connections " +
|
||||
"ORDER BY connections DESC");
|
||||
List<Map<String, Object>> multiHospital = new ArrayList<>();
|
||||
result3.list().forEach(r -> multiHospital.add(r.asMap()));
|
||||
report.setMultiHospitalReport(multiHospital);
|
||||
|
||||
// 4. Overall counts
|
||||
var nodeCountResult = session.run(
|
||||
"MATCH (n) RETURN labels(n)[0] AS label, count(*) AS count ORDER BY count DESC");
|
||||
Map<String, Long> nodeCounts = new HashMap<>();
|
||||
nodeCountResult.list().forEach(r ->
|
||||
nodeCounts.put(r.get("label").asString(), r.get("count").asLong()));
|
||||
report.setNodeCounts(nodeCounts);
|
||||
|
||||
var relCountResult = session.run(
|
||||
"MATCH ()-[rel]->() RETURN type(rel) AS type, count(*) AS count ORDER BY count DESC");
|
||||
Map<String, Long> relCounts = new HashMap<>();
|
||||
relCountResult.list().forEach(r ->
|
||||
relCounts.put(r.get("type").asString(), r.get("count").asLong()));
|
||||
report.setRelCounts(relCounts);
|
||||
|
||||
var totalNodes = session.run("MATCH (n) RETURN count(*) AS total").single().get("total").asLong();
|
||||
report.setTotalNodes(totalNodes);
|
||||
|
||||
var totalRels = session.run("MATCH ()-[rel]->() RETURN count(*) AS total").single().get("total").asLong();
|
||||
report.setTotalRelationships(totalRels);
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
public void exportOverlapCsv(Path outputPath, List<Map<String, Object>> overlaps) throws IOException {
|
||||
StringBuilder csv = new StringBuilder();
|
||||
csv.append("osoba;szpitale;mandaty;afiliacje;confidence;sourceUrls\n");
|
||||
for (Map<String, Object> row : overlaps) {
|
||||
String person = (String) row.getOrDefault("person", "");
|
||||
List<?> szpitale = (List<?>) row.getOrDefault("szpitale", List.of());
|
||||
List<?> mandaty = (List<?>) row.getOrDefault("mandaty", List.of());
|
||||
csv.append(person)
|
||||
.append(";").append(String.join(", ", szpitale.stream().map(Object::toString).toList()))
|
||||
.append(";").append(String.join(", ", mandaty.stream().map(Object::toString).toList()))
|
||||
.append("\n");
|
||||
}
|
||||
Files.writeString(outputPath, csv.toString());
|
||||
log.info("Overlap report written to: {}", outputPath);
|
||||
}
|
||||
|
||||
public static class ValidationReport {
|
||||
public List<Map<String, Object>> hospitalsWithoutDirector = new ArrayList<>();
|
||||
public List<Map<String, Object>> overlapReport = new ArrayList<>();
|
||||
public List<Map<String, Object>> multiHospitalReport = new ArrayList<>();
|
||||
public Map<String, Long> nodeCounts = new HashMap<>();
|
||||
public Map<String, Long> relCounts = new HashMap<>();
|
||||
public long totalNodes;
|
||||
public long totalRelationships;
|
||||
|
||||
public void setHospitalsWithoutDirector(List<Map<String, Object>> list) { this.hospitalsWithoutDirector = list; }
|
||||
public void setOverlapReport(List<Map<String, Object>> list) { this.overlapReport = list; }
|
||||
public void setMultiHospitalReport(List<Map<String, Object>> list) { this.multiHospitalReport = list; }
|
||||
public void setNodeCounts(Map<String, Long> map) { this.nodeCounts = map; }
|
||||
public void setRelCounts(Map<String, Long> map) { this.relCounts = map; }
|
||||
public void setTotalNodes(long n) { this.totalNodes = n; }
|
||||
public void setTotalRelationships(long r) { this.totalRelationships = r; }
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Validation report:\n" +
|
||||
" Total nodes: " + totalNodes + "\n" +
|
||||
" Total relationships: " + totalRelationships + "\n" +
|
||||
" Node types: " + nodeCounts + "\n" +
|
||||
" Relationship types: " + relCounts + "\n" +
|
||||
" Hospitals without directors: " + hospitalsWithoutDirector.size() + "\n" +
|
||||
" People with mandates: " + overlapReport.size() + "\n" +
|
||||
" People in multiple hospitals: " + multiHospitalReport.size() + "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.developx.szpitale.load;
|
||||
|
||||
import org.neo4j.driver.Driver;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
public class LoadCommand {
|
||||
|
||||
private final Driver neo4jDriver;
|
||||
|
||||
public LoadCommand(Driver neo4jDriver) {
|
||||
this.neo4jDriver = neo4jDriver;
|
||||
}
|
||||
|
||||
public String load(String inDir, String voivodeship, boolean createSchema, boolean wipe, boolean validate) {
|
||||
try {
|
||||
if (wipe) {
|
||||
new GraphSchema(neo4jDriver).dropAll();
|
||||
System.out.println("Graph wiped.");
|
||||
}
|
||||
if (createSchema) {
|
||||
new GraphSchema(neo4jDriver).createConstraints();
|
||||
System.out.println("Schema created.");
|
||||
}
|
||||
|
||||
CanonicalReader reader = new CanonicalReader();
|
||||
var datasets = reader.readAll(Path.of(inDir), voivodeship);
|
||||
if (datasets.isEmpty()) {
|
||||
return "Error: No canonical datasets found for voivodeship: " + voivodeship;
|
||||
}
|
||||
|
||||
GraphLoader loader = new GraphLoader(neo4jDriver);
|
||||
GraphLoader.LoadStats stats = loader.load(datasets);
|
||||
System.out.println(stats);
|
||||
|
||||
if (validate) {
|
||||
GraphValidator validator = new GraphValidator(neo4jDriver);
|
||||
GraphValidator.ValidationReport report = validator.validate();
|
||||
System.out.println(report);
|
||||
|
||||
try {
|
||||
validator.exportOverlapCsv(Path.of("overlap_report.csv"), report.overlapReport);
|
||||
} catch (IOException e) {
|
||||
System.err.println("Error writing overlap report: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return "Load completed successfully.";
|
||||
} catch (Exception e) {
|
||||
return "Error during load: " + e.getMessage();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.developx.szpitale.model.enums.AffiliationType;
|
||||
import com.developx.szpitale.model.enums.ConfidenceLevel;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record Affiliation(
|
||||
@JsonProperty("personId") String personId,
|
||||
@JsonProperty("type") AffiliationType type,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("confidence") ConfidenceLevel confidence,
|
||||
@JsonProperty("note") String note,
|
||||
@JsonProperty("sourceUrls") List<String> sourceUrls
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
|
||||
public record CanonicalDataset(
|
||||
@JsonProperty("schemaVersion") String schemaVersion,
|
||||
@JsonProperty("voivodeship") String voivodeship,
|
||||
@JsonProperty("generatedAt") LocalDateTime generatedAt,
|
||||
@JsonProperty("hospitals") List<Hospital> hospitals,
|
||||
@JsonProperty("people") List<Person> people,
|
||||
@JsonProperty("affiliations") List<Affiliation> affiliations,
|
||||
@JsonProperty("roles") List<Role> roles,
|
||||
@JsonProperty("mandates") List<Mandate> mandates
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.developx.szpitale.model.enums.LegalForm;
|
||||
import com.developx.szpitale.model.enums.SupervisoryBodyType;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record Hospital(
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("shortName") String shortName,
|
||||
@JsonProperty("city") String city,
|
||||
@JsonProperty("voivodeship") String voivodeship,
|
||||
@JsonProperty("legalForm") LegalForm legalForm,
|
||||
@JsonProperty("foundingBody") String foundingBody,
|
||||
@JsonProperty("supervisoryBodyType") SupervisoryBodyType supervisoryBodyType,
|
||||
@JsonProperty("nip") String nip,
|
||||
@JsonProperty("krs") String krs,
|
||||
@JsonProperty("website") String website,
|
||||
@JsonProperty("sourceUrls") List<String> sourceUrls
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.developx.szpitale.model.enums.MandateType;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record Mandate(
|
||||
@JsonProperty("personId") String personId,
|
||||
@JsonProperty("mandateType") MandateType mandateType,
|
||||
@JsonProperty("body") String body,
|
||||
@JsonProperty("term") String term,
|
||||
@JsonProperty("voivodeship") String voivodeship,
|
||||
@JsonProperty("sourceUrls") List<String> sourceUrls
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record Person(
|
||||
@JsonProperty("id") String id,
|
||||
@JsonProperty("fullName") String fullName,
|
||||
@JsonProperty("displayName") String displayName,
|
||||
@JsonProperty("titles") List<String> titles,
|
||||
@JsonProperty("sourceUrls") List<String> sourceUrls
|
||||
) {
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public record RawHospitalRecord(
|
||||
String hospitalName,
|
||||
String shortName,
|
||||
String city,
|
||||
String voivodeship,
|
||||
String legalForm,
|
||||
String foundingBody,
|
||||
String supervisoryBodyType,
|
||||
String nip,
|
||||
String krs,
|
||||
String website,
|
||||
List<RawPersonEntry> persons
|
||||
) {
|
||||
|
||||
public record RawPersonEntry(
|
||||
String function,
|
||||
String fullName,
|
||||
String displayTitle,
|
||||
String partyAffiliation,
|
||||
String confidence,
|
||||
String sourceUrl,
|
||||
String organ,
|
||||
String statusNote
|
||||
) {
|
||||
}
|
||||
|
||||
public record RawMandate(
|
||||
String personName,
|
||||
String mandateType,
|
||||
String body,
|
||||
String term,
|
||||
String voivodeship,
|
||||
String sourceUrl
|
||||
) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.developx.szpitale.model;
|
||||
|
||||
import com.developx.szpitale.model.enums.OrganType;
|
||||
import com.developx.szpitale.model.enums.RoleStatus;
|
||||
import com.developx.szpitale.model.enums.RoleType;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
|
||||
public record Role(
|
||||
@JsonProperty("hospitalId") String hospitalId,
|
||||
@JsonProperty("personId") String personId,
|
||||
@JsonProperty("roleType") RoleType roleType,
|
||||
@JsonProperty("roleLabel") String roleLabel,
|
||||
@JsonProperty("organ") OrganType organ,
|
||||
@JsonProperty("status") RoleStatus status,
|
||||
@JsonProperty("validFrom") LocalDate validFrom,
|
||||
@JsonProperty("validTo") LocalDate validTo,
|
||||
@JsonProperty("note") String note,
|
||||
@JsonProperty("sourceUrls") List<String> sourceUrls
|
||||
) {
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum AffiliationType {
|
||||
PARTIA,
|
||||
KOMITET_WYBORCZY,
|
||||
KWW_LOKALNY
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum ConfidenceLevel {
|
||||
POTWIERDZONA,
|
||||
NIEPOTWIERDZONA,
|
||||
NIEZWERYFIKOWANA,
|
||||
BRAK_DANYCH
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum LegalForm {
|
||||
SPZOZ,
|
||||
SP_PSYCHIATRYCZNY_ZOZ,
|
||||
SPOLKA_Z_OO,
|
||||
INNY
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum MandateType {
|
||||
RADNY_SEJMIKU,
|
||||
RADNY_POWIATU,
|
||||
RADNY_GMINY,
|
||||
WOJT_BURMISTRZ_PREZYDENT,
|
||||
STAROSTA,
|
||||
POSEL,
|
||||
SENATOR,
|
||||
MARSZALEK
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum OrganType {
|
||||
DYREKCJA,
|
||||
RADA_SPOLECZNA,
|
||||
RADA_NADZORCZA
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum RoleStatus {
|
||||
AKTUALNY,
|
||||
PELNIACY_OBOWIAZKI,
|
||||
ELEKT,
|
||||
BYLY
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum RoleType {
|
||||
DYREKTOR,
|
||||
ZASTEPCA_DYREKTORA,
|
||||
GLOWNY_KSIEGOWY,
|
||||
PRZELOZONA_PIELEGNIARKOW,
|
||||
CZLONEK_ORGANU_NADZORCZEGO,
|
||||
PRZEWODNICZACY_ORGANU,
|
||||
SEKRETARZ_ORGANU,
|
||||
Z_CA_PRZEWODNICZACEGO
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
package com.developx.szpitale.model.enums;
|
||||
|
||||
public enum SupervisoryBodyType {
|
||||
RADA_SPOLECZNA,
|
||||
RADA_NADZORCZA
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
spring:
|
||||
neo4j:
|
||||
uri: bolt://localhost:7687
|
||||
authentication:
|
||||
username: neo4j
|
||||
password: password
|
||||
application:
|
||||
name: szpitale-graph
|
||||
|
||||
app:
|
||||
ingest:
|
||||
canonical-dir: canonical
|
||||
neo4j:
|
||||
batch-size: 1000
|
||||
Reference in New Issue
Block a user