T1: Deduplicator compile fix + unit tests [checkpoint]

This commit is contained in:
Artur Kruszewski
2026-07-07 22:20:08 +02:00
commit 38a93d0c4d
114 changed files with 6942 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
wrapperVersion=3.3.4
distributionType=only-script
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.12/apache-maven-3.9.12-bin.zip
+18
View File
@@ -0,0 +1,18 @@
services:
neo4j:
image: neo4j:5
container_name: neo4j-szpitale
ports:
- "7474:7474"
- "7687:7687"
environment:
- NEO4J_AUTH=neo4j/password
- NEO4J_apoc_export_file_enabled=true
- NEO4J_apoc_import_file_enabled=true
- NEO4J_apoc_import_file_use__neo4j__config=true
volumes:
- neo4j-data:/data
restart: unless-stopped
volumes:
neo4j-data:
+18
View File
@@ -0,0 +1,18 @@
{
"schemaVersion": "1.0",
"project": "szpitale-graph",
"description": "Stan wykonania kolejki zadań lokalnego agenta (PLAN2.md §11-13). Zrodlo prawdy przy konflikcie: git log.",
"resumeHint": "Wczytaj ten plik, znajdz pierwszy task o statusie todo/in_progress. Jesli in_progress i drzewo git brudne -> git checkout -- . && git clean -fd, ustaw task na todo, zacznij od zera (PLAN2.md §13.1).",
"createdAt": "2026-07-07",
"updatedAt": "2026-07-07",
"tasks": [
{ "id": "T1", "title": "Deduplicator compile fix", "tag": null, "status": "in_progress", "startedAt": "2026-07-07T00:00:00Z", "finishedAt": null, "note": "BLOKER: build czerwony az to zrobione" },
{ "id": "T2", "title": "NameNormalizer testy", "tag": null, "status": "todo", "startedAt": null, "finishedAt": null, "note": "" },
{ "id": "T3", "title": "PartyNormalizer testy", "tag": null, "status": "todo", "startedAt": null, "finishedAt": null, "note": "" },
{ "id": "T4", "title": "MarkdownRegistrySource test na fixture", "tag": null, "status": "todo", "startedAt": null, "finishedAt": null, "note": "" },
{ "id": "T5", "title": "Canonical write/read round-trip", "tag": null, "status": "todo", "startedAt": null, "finishedAt": null, "note": "" },
{ "id": "T6", "title": "GraphLoader idempotencja", "tag": "NEO4J", "status": "todo", "startedAt": null, "finishedAt": null, "note": "wymaga Dockera; bez -> skipped" },
{ "id": "T7", "title": "GraphValidator + overlap_report.csv", "tag": "NEO4J", "status": "todo", "startedAt": null, "finishedAt": null, "note": "wymaga Dockera; bez -> skipped" },
{ "id": "T8", "title": "Zielony pelny build", "tag": null, "status": "todo", "startedAt": null, "finishedAt": null, "note": "" }
]
}
Vendored Executable
+295
View File
@@ -0,0 +1,295 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Apache Maven Wrapper startup batch script, version 3.3.4
#
# Optional ENV vars
# -----------------
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
# MVNW_REPOURL - repo url base for downloading maven distribution
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
# ----------------------------------------------------------------------------
set -euf
[ "${MVNW_VERBOSE-}" != debug ] || set -x
# OS specific support.
native_path() { printf %s\\n "$1"; }
case "$(uname)" in
CYGWIN* | MINGW*)
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
native_path() { cygpath --path --windows "$1"; }
;;
esac
# set JAVACMD and JAVACCMD
set_java_home() {
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
if [ -n "${JAVA_HOME-}" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
JAVACCMD="$JAVA_HOME/jre/sh/javac"
else
JAVACMD="$JAVA_HOME/bin/java"
JAVACCMD="$JAVA_HOME/bin/javac"
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
return 1
fi
fi
else
JAVACMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v java
)" || :
JAVACCMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v javac
)" || :
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
return 1
fi
fi
}
# hash string like Java String::hashCode
hash_string() {
str="${1:-}" h=0
while [ -n "$str" ]; do
char="${str%"${str#?}"}"
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
str="${str#?}"
done
printf %x\\n $h
}
verbose() { :; }
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
die() {
printf %s\\n "$1" >&2
exit 1
}
trim() {
# MWRAPPER-139:
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
# Needed for removing poorly interpreted newline sequences when running in more
# exotic environments such as mingw bash on Windows.
printf "%s" "${1}" | tr -d '[:space:]'
}
scriptDir="$(dirname "$0")"
scriptName="$(basename "$0")"
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
while IFS="=" read -r key value; do
case "${key-}" in
distributionUrl) distributionUrl=$(trim "${value-}") ;;
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
esac
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
case "${distributionUrl##*/}" in
maven-mvnd-*bin.*)
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
*)
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
distributionPlatform=linux-amd64
;;
esac
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
;;
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
esac
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
distributionUrlName="${distributionUrl##*/}"
distributionUrlNameMain="${distributionUrlName%.*}"
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
exec_maven() {
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
}
if [ -d "$MAVEN_HOME" ]; then
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
exec_maven "$@"
fi
case "${distributionUrl-}" in
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
esac
# prepare tmp dir
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
trap clean HUP INT TERM EXIT
else
die "cannot create temp dir"
fi
mkdir -p -- "${MAVEN_HOME%/*}"
# Download and Install Apache Maven
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
verbose "Downloading from: $distributionUrl"
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
# select .zip or .tar.gz
if ! command -v unzip >/dev/null; then
distributionUrl="${distributionUrl%.zip}.tar.gz"
distributionUrlName="${distributionUrl##*/}"
fi
# verbose opt
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
# normalize http auth
case "${MVNW_PASSWORD:+has-password}" in
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
esac
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
verbose "Found wget ... using wget"
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
verbose "Found curl ... using curl"
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
elif set_java_home; then
verbose "Falling back to use Java to download"
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
cat >"$javaSource" <<-END
public class Downloader extends java.net.Authenticator
{
protected java.net.PasswordAuthentication getPasswordAuthentication()
{
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
}
public static void main( String[] args ) throws Exception
{
setDefault( new Downloader() );
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
}
}
END
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
verbose " - Compiling Downloader.java ..."
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
verbose " - Running Downloader.java ..."
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
fi
# If specified, validate the SHA-256 sum of the Maven distribution zip file
if [ -n "${distributionSha256Sum-}" ]; then
distributionSha256Result=false
if [ "$MVN_CMD" = mvnd.sh ]; then
echo "Checksum validation is not supported for maven-mvnd." >&2
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
elif command -v sha256sum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
distributionSha256Result=true
fi
elif command -v shasum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
distributionSha256Result=true
fi
else
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
fi
if [ $distributionSha256Result = false ]; then
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
exit 1
fi
fi
# unzip and move
if command -v unzip >/dev/null; then
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
else
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
fi
# Find the actual extracted directory name (handles snapshots where filename != directory name)
actualDistributionDir=""
# First try the expected directory name (for regular distributions)
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
actualDistributionDir="$distributionUrlNameMain"
fi
fi
# If not found, search for any directory with the Maven executable (for snapshots)
if [ -z "$actualDistributionDir" ]; then
# enable globbing to iterate over items
set +f
for dir in "$TMP_DOWNLOAD_DIR"/*; do
if [ -d "$dir" ]; then
if [ -f "$dir/bin/$MVN_CMD" ]; then
actualDistributionDir="$(basename "$dir")"
break
fi
fi
done
set -f
fi
if [ -z "$actualDistributionDir" ]; then
verbose "Contents of $TMP_DOWNLOAD_DIR:"
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
die "Could not find Maven distribution directory in extracted archive"
fi
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
clean || :
exec_maven "$@"
+189
View File
@@ -0,0 +1,189 @@
<# : batch portion
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Apache Maven Wrapper startup batch script, version 3.3.4
@REM
@REM Optional ENV vars
@REM MVNW_REPOURL - repo url base for downloading maven distribution
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
@REM ----------------------------------------------------------------------------
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
@SET __MVNW_CMD__=
@SET __MVNW_ERROR__=
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
@SET PSModulePath=
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
)
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
@SET __MVNW_PSMODULEP_SAVE=
@SET __MVNW_ARG0_NAME__=
@SET MVNW_USERNAME=
@SET MVNW_PASSWORD=
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
@echo Cannot start maven from wrapper >&2 && exit /b 1
@GOTO :EOF
: end batch / begin powershell #>
$ErrorActionPreference = "Stop"
if ($env:MVNW_VERBOSE -eq "true") {
$VerbosePreference = "Continue"
}
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
if (!$distributionUrl) {
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
}
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
"maven-mvnd-*" {
$USE_MVND = $true
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
$MVN_CMD = "mvnd.cmd"
break
}
default {
$USE_MVND = $false
$MVN_CMD = $script -replace '^mvnw','mvn'
break
}
}
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
if ($env:MVNW_REPOURL) {
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
}
$distributionUrlName = $distributionUrl -replace '^.*/',''
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
$MAVEN_M2_PATH = "$HOME/.m2"
if ($env:MAVEN_USER_HOME) {
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
}
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
}
$MAVEN_WRAPPER_DISTS = $null
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
} else {
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
}
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
exit $?
}
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
}
# prepare tmp dir
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
trap {
if ($TMP_DOWNLOAD_DIR.Exists) {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
}
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
# Download and Install Apache Maven
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
Write-Verbose "Downloading from: $distributionUrl"
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
$webclient = New-Object System.Net.WebClient
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
}
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
# If specified, validate the SHA-256 sum of the Maven distribution zip file
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
if ($distributionSha256Sum) {
if ($USE_MVND) {
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
}
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
}
}
# unzip and move
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
# Find the actual extracted directory name (handles snapshots where filename != directory name)
$actualDistributionDir = ""
# First try the expected directory name (for regular distributions)
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
$actualDistributionDir = $distributionUrlNameMain
}
# If not found, search for any directory with the Maven executable (for snapshots)
if (!$actualDistributionDir) {
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
if (Test-Path -Path $testPath -PathType Leaf) {
$actualDistributionDir = $_.Name
}
}
}
if (!$actualDistributionDir) {
Write-Error "Could not find Maven distribution directory in extracted archive"
}
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
try {
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
} catch {
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
Write-Error "fail to move MAVEN_HOME"
}
} finally {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+114
View File
@@ -0,0 +1,114 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.4.4</version>
<relativePath/>
</parent>
<groupId>com.developx</groupId>
<artifactId>szpitale-graph</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>szpitale-graph</name>
<description>Neo4j graph application for Polish hospitals data</description>
<properties>
<java.version>21</java.version>
</properties>
<dependencies>
<!-- Spring Boot -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<!-- Spring Shell -->
<!-- Neo4j Spring Boot starter (version managed by parent BOM) -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-neo4j</artifactId>
</dependency>
<!-- Jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<!-- Jsoup (HTML scraping) -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.18.3</version>
</dependency>
<!-- Commons Text (fuzzy matching) -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.12.0</version>
</dependency>
<!-- Commons CSV (reports) -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.12.0</version>
</dependency>
<!-- Lombok -->
<!-- Testing -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>neo4j</artifactId>
<version>1.20.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.20.4</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
</configuration>
</plugin>
</plugins>
</build>
</project>
@@ -0,0 +1,12 @@
package com.developx.szpitale;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class SzpitaleGraphApplication {
public static void main(String[] args) {
SpringApplication.run(SzpitaleGraphApplication.class, args);
}
}
@@ -0,0 +1,213 @@
package com.developx.szpitale.ingest;
import com.developx.szpitale.model.*;
import com.developx.szpitale.model.enums.*;
import com.developx.szpitale.ingest.normalize.NameNormalizer;
import com.developx.szpitale.ingest.normalize.PartyNormalizer;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class CanonicalWriter {
private final ObjectMapper objectMapper = new ObjectMapper();
private final NameNormalizer nameNormalizer = new NameNormalizer();
private final PartyNormalizer partyNormalizer = new PartyNormalizer();
public CanonicalWriter() {
objectMapper.registerModule(new JavaTimeModule());
objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
}
public void write(Path outputDir, List<RawHospitalRecord> rawRecords, String voivodeship,
IngestResults results) throws IOException {
Path canonicalDir = outputDir.resolve("canonical").normalize();
Files.createDirectories(canonicalDir);
List<Hospital> hospitals = new ArrayList<>();
List<Person> people = new ArrayList<>();
List<Role> roles = new ArrayList<>();
List<Affiliation> affiliations = new ArrayList<>();
for (RawHospitalRecord rawRecord : rawRecords) {
String hospitalId = nameNormalizer.makeHospitalId(voivodeship, rawRecord.hospitalName());
hospitals.add(new Hospital(
hospitalId,
rawRecord.hospitalName(),
rawRecord.shortName(),
rawRecord.city(),
voivodeship,
parseLegalForm(rawRecord.legalForm()),
rawRecord.foundingBody(),
parseSupervisoryBodyType(rawRecord.supervisoryBodyType()),
rawRecord.nip(),
rawRecord.krs(),
rawRecord.website(),
extractUniqueUrls(rawRecord.persons())
));
Map<String, Person> peopleMap = new LinkedHashMap<>();
Map<String, Set<String>> personUrlsMap = new LinkedHashMap<>();
for (RawHospitalRecord.RawPersonEntry personEntry : rawRecord.persons()) {
String fullName = personEntry.fullName();
if (fullName.isEmpty()) continue;
if (fullName.toLowerCase().contains("brak") &&
(personEntry.sourceUrl() == null || personEntry.sourceUrl().toLowerCase().contains("brak"))) {
continue;
}
String personId = nameNormalizer.makePersonId(fullName);
List<String> titles = nameNormalizer.extractTitles(fullName);
String extractedName = nameNormalizer.extractFullName(fullName);
if (!peopleMap.containsKey(personId)) {
peopleMap.put(personId, new Person(
personId, extractedName, personEntry.displayTitle(),
titles, new ArrayList<>()
));
personUrlsMap.put(personId, new LinkedHashSet<>());
}
if (personEntry.sourceUrl() != null && !personEntry.sourceUrl().isEmpty()) {
personUrlsMap.get(personId).add(personEntry.sourceUrl());
}
RoleType roleType = parseRoleType(personEntry.function());
OrganType organ = parseOrganType(personEntry.organ());
RoleStatus status = parseStatus(personEntry.statusNote());
roles.add(new Role(
hospitalId, personId, roleType,
personEntry.function(),
organ, status, null, null,
null,
List.of(personEntry.sourceUrl())
));
String affText = personEntry.partyAffiliation();
if (affText != null && !affText.isEmpty() &&
!affText.toLowerCase().contains("brak") &&
!affText.toLowerCase().contains("bezpartyj")) {
String canonicalParty = partyNormalizer.canonicalize(affText);
AffiliationType affType = partyNormalizer.determineType(affText);
ConfidenceLevel confidence = partyNormalizer.mapConfidence(personEntry.confidence());
String note = affText.contains("historycznie")
? "historycznie: " + affText
: affText;
affiliations.add(new Affiliation(
personId, affType, canonicalParty, confidence,
note,
List.of(personEntry.sourceUrl())
));
}
}
// Update people with merged URLs
List<Person> updatedPeople = new ArrayList<>();
for (Map.Entry<String, Set<String>> entry : personUrlsMap.entrySet()) {
Person p = peopleMap.get(entry.getKey());
if (p != null) {
updatedPeople.add(new Person(
p.id(), p.fullName(), p.displayName(),
p.titles(), new ArrayList<>(entry.getValue())
));
}
}
people.addAll(updatedPeople);
}
CanonicalDataset dataset = new CanonicalDataset(
"1.0", voivodeship, LocalDateTime.now(),
hospitals, people, affiliations, roles, List.of()
);
Path outputFile = canonicalDir.resolve(voivodeship + ".json");
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), dataset);
results.addHospitals(hospitals.size());
results.addPeople(people.size());
results.addRoles(roles.size());
results.addAffiliations(affiliations.size());
}
private List<String> extractUniqueUrls(List<RawHospitalRecord.RawPersonEntry> persons) {
return persons.stream()
.map(RawHospitalRecord.RawPersonEntry::sourceUrl)
.filter(url -> url != null && !url.isEmpty() && !url.toLowerCase().contains("brak"))
.distinct()
.collect(Collectors.toList());
}
private LegalForm parseLegalForm(String form) {
if (form == null) return LegalForm.SPZOZ;
String upper = form.toUpperCase();
if (upper.contains("SPOLKA")) return LegalForm.SPOLKA_Z_OO;
if (upper.contains("PSYCHIATRYCZNY")) return LegalForm.SP_PSYCHIATRYCZNY_ZOZ;
return LegalForm.SPZOZ;
}
private SupervisoryBodyType parseSupervisoryBodyType(String type) {
if (type == null) return SupervisoryBodyType.RADA_SPOLECZNA;
return "NADZORCZA".equalsIgnoreCase(type)
? SupervisoryBodyType.RADA_NADZORCZA
: SupervisoryBodyType.RADA_SPOLECZNA;
}
private RoleType parseRoleType(String function) {
String lower = function.toLowerCase();
if (lower.contains("dyrektor nacz")) {
return RoleType.DYREKTOR;
}
if (lower.contains("dyrektor") && !lower.contains("z-ca") && !lower.contains("zast")) {
return RoleType.DYREKTOR;
}
if (lower.contains("z-ca") || lower.contains("zast")) {
return RoleType.ZASTEPCA_DYREKTORA;
}
if (lower.contains("glown") || lower.contains("gl. ")) {
return RoleType.GLOWNY_KSIEGOWY;
}
if (lower.contains("naczel") && lower.contains("pieleg")) {
return RoleType.PRZELOZONA_PIELEGNIARKOW;
}
if (lower.contains("przewodnicz") && !lower.contains("z-ca przewodnich")) {
return RoleType.PRZEWODNICZACY_ORGANU;
}
if (lower.contains("sekte")) {
return RoleType.SEKRETARZ_ORGANU;
}
if (lower.contains("z-ca")) {
return RoleType.Z_CA_PRZEWODNICZACEGO;
}
return RoleType.CZLONEK_ORGANU_NADZORCZEGO;
}
private OrganType parseOrganType(String organ) {
if (organ == null) return OrganType.RADA_SPOLECZNA;
String upper = organ.toUpperCase();
if (upper.contains("DYREKCJA")) return OrganType.DYREKCJA;
if (upper.contains("NADZORCZA")) return OrganType.RADA_NADZORCZA;
return OrganType.RADA_SPOLECZNA;
}
private RoleStatus parseStatus(String note) {
if (note == null) return RoleStatus.AKTUALNY;
String lower = note.toLowerCase();
if (lower.contains("pelniac") || lower.contains("p.o.")) return RoleStatus.PELNIACY_OBOWIAZKI;
if (lower.contains("elekt")) return RoleStatus.ELEKT;
return RoleStatus.AKTUALNY;
}
}
@@ -0,0 +1,85 @@
package com.developx.szpitale.ingest;
import com.developx.szpitale.ingest.normalize.Deduplicator;
import com.developx.szpitale.ingest.source.MarkdownRegistrySource;
import com.developx.szpitale.ingest.source.VoivodeshipSource;
import com.developx.szpitale.model.RawHospitalRecord;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
public class IngestCommand {
public String ingest(String voivodeship, String source, String outDir, boolean split) {
if (!"markdown".equals(source)) {
return "Error: Only 'markdown' source is currently supported.";
}
Path outputDir = Path.of(outDir).toAbsolutePath().normalize();
Path researchDir = Path.of("data").toAbsolutePath().normalize();
List<VoivodeshipSource> sources = MarkdownRegistrySource.discoverAllSources(researchDir);
if (sources.isEmpty()) {
return "Error: No research-*.{md,markdown} files found in data/ directory.";
}
if (!"all".equals(voivodeship)) {
String target = voivodeship.toLowerCase();
sources = sources.stream()
.filter(s -> s.voivodeship().equals(target))
.collect(Collectors.toList());
if (sources.isEmpty()) {
return "Warning: No source found for voivodeship: " + voivodeship;
}
}
IngestResults totalResults = new IngestResults();
CanonicalWriter writer = new CanonicalWriter();
for (VoivodeshipSource src : sources) {
System.out.println("\nProcessing voivodeship: " + src.voivodeship());
List<RawHospitalRecord> records = src.fetch();
System.out.println(" Found " + records.size() + " hospital records");
IngestResults results = new IngestResults();
try {
writer.write(outputDir, records, src.voivodeship(), results);
System.out.println(" Written: " + outputDir.resolve("canonical").resolve(src.voivodeship() + ".json"));
} catch (IOException e) {
System.err.println(" Error writing: " + e.getMessage());
}
totalResults.addHospitals(results.getHospitalCount());
totalResults.addPeople(results.getPersonCount());
totalResults.addRoles(results.getRoleCount());
totalResults.addAffiliations(results.getAffiliationCount());
}
try {
writeIngestReport(outputDir, totalResults, sources);
} catch (IOException e) {
System.err.println("Error writing ingest report: " + e.getMessage());
}
return totalResults.toString();
}
private void writeIngestReport(Path outputDir, IngestResults results, List<VoivodeshipSource> sources) throws IOException {
Path reportPath = outputDir.resolve("ingest-report.json");
String report = "{" +
"\"schemaVersion\":\"1.0\"," +
"\"generatedAt\":\"" + java.time.LocalDateTime.now() + "\"," +
"\"voivodeshipsProcessed\":" + sources.size() + "," +
"\"hospitals\":" + results.getHospitalCount() + "," +
"\"people\":" + results.getPersonCount() + "," +
"\"roles\":" + results.getRoleCount() + "," +
"\"affiliations\":" + results.getAffiliationCount() + "," +
"\"gaps\":" + results.getGapCount() +
"}";
Files.writeString(reportPath, report);
System.out.println("\nIngest report written to: " + reportPath);
}
}
@@ -0,0 +1,29 @@
package com.developx.szpitale.ingest;
public class IngestResults {
private int hospitalCount = 0;
private int personCount = 0;
private int roleCount = 0;
private int affiliationCount = 0;
private int gapCount = 0;
public void addHospitals(int count) { hospitalCount += count; }
public void addPeople(int count) { personCount += count; }
public void addRoles(int count) { roleCount += count; }
public void addAffiliations(int count) { affiliationCount += count; }
public void addGaps(int count) { gapCount += count; }
public int getHospitalCount() { return hospitalCount; }
public int getPersonCount() { return personCount; }
public int getRoleCount() { return roleCount; }
public int getAffiliationCount() { return affiliationCount; }
public int getGapCount() { return gapCount; }
@Override
public String toString() {
return String.format(
"Ingest results: %d hospitals, %d people, %d roles, %d affiliations, %d gaps",
hospitalCount, personCount, roleCount, affiliationCount, gapCount
);
}
}
@@ -0,0 +1,82 @@
package com.developx.szpitale.ingest.normalize;
import com.developx.szpitale.model.Person;
import org.apache.commons.text.similarity.JaroWinklerSimilarity;
import java.util.*;
import java.util.stream.Collectors;
public class Deduplicator {
private static final double DUPLICATE_THRESHOLD = 0.85;
private static final double SUSPECT_THRESHOLD = 0.70;
private final NameNormalizer nameNormalizer = new NameNormalizer();
public List<Person> deduplicate(List<Person> people) {
List<String> personIds = new ArrayList<>();
Map<String, List<Person>> bySlug = people.stream()
.collect(Collectors.groupingBy(Person::id));
List<Person> result = new ArrayList<>();
Set<String> processed = new HashSet<>();
for (Map.Entry<String, List<Person>> entry : bySlug.entrySet()) {
List<Person> group = entry.getValue();
if (group.size() == 1) {
result.add(group.get(0));
} else {
// Same slug - merge source URLs
Person merged = mergeBySlug(group);
result.add(merged);
}
}
// Check cross-slug candidates with fuzzy matching
checkFuzzyDuplicates(result);
return result;
}
private Person mergeBySlug(List<Person> duplicates) {
Person first = duplicates.get(0);
Set<String> allUrls = duplicates.stream()
.flatMap(p -> p.sourceUrls().stream())
.collect(Collectors.toSet());
return new Person(
first.id(),
first.fullName(),
first.displayName(),
first.titles(),
new ArrayList<>(allUrls)
);
}
private void checkFuzzyDuplicates(List<Person> people) {
Map<String, List<String>> groups = new HashMap<>();
for (int i = 0; i < people.size(); i++) {
Person person = people.get(i);
List<String> candidates = new ArrayList<>();
for (int j = 0; j < i; j++) {
Person existing = people.get(j);
double similarity = nameNormalizer.fuzzySimilarity(
person.fullName(),
existing.fullName()
);
if (similarity >= SUSPECT_THRESHOLD && similarity < DUPLICATE_THRESHOLD) {
candidates.add(existing.id());
}
}
if (!candidates.isEmpty()) {
groups.put(person.id(), candidates);
}
}
if (!groups.isEmpty()) {
System.out.println("Warning: Possible name similarity conflicts (manual review recommended):");
groups.forEach((id, candidates) ->
System.out.println(" " + id + " -> similar to: " + String.join(", ", candidates)));
}
}
}
@@ -0,0 +1,73 @@
package com.developx.szpitale.ingest.normalize;
import org.apache.commons.text.similarity.JaroWinklerSimilarity;
import java.text.Normalizer;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class NameNormalizer {
private static final Set<String> TITLE_TOKENS = Set.of(
"prof.", "prof", "dr hab.", "dr.hab.", "dr.", "mgr", "mgr in\u017c.", "in\u017c.",
"licencjat in\u017c.", "lek.", "lek.med.", "n. med.", "n. o zdr."
);
private final JaroWinklerSimilarity jaroWinkler = new JaroWinklerSimilarity();
public String makeSlug(String fullName) {
String normalized = Normalizer.normalize(fullName, Normalizer.Form.NFKD);
normalized = normalized.replaceAll("\\p{M}", "");
normalized = normalized.replaceAll("[^a-zA-Z0-9\\s-]", " ");
normalized = normalized.toLowerCase().replaceAll("\\s+", "-").replaceAll("-+", "-");
normalized = normalized.replaceAll("^-|-$", "");
return normalized;
}
public String extractFullName(String rawName) {
return extractGivenAndFamilyNames(rawName);
}
public List<String> extractTitles(String rawName) {
List<String> titles = new ArrayList<>();
String lower = rawName.toLowerCase();
for (String title : TITLE_TOKENS) {
if (lower.contains(title.toLowerCase())) {
titles.add(title.trim());
}
}
return titles;
}
public String makePersonId(String fullName) {
String slug = makeSlug(fullName);
return "person:" + slug;
}
public String makeHospitalId(String voivodeship, String hospitalName) {
String slug = makeSlug(hospitalName);
return "hosp:" + voivodeship + ":" + slug;
}
public double fuzzySimilarity(String a, String b) {
return jaroWinkler.apply(a.toLowerCase(), b.toLowerCase());
}
public List<String> findDuplicateCandidates(List<String> names, String targetName, double threshold) {
return names.stream()
.filter(name -> fuzzySimilarity(name, targetName) >= threshold)
.filter(name -> !name.equals(targetName))
.collect(Collectors.toList());
}
private String extractGivenAndFamilyNames(String rawName) {
Pattern pattern = Pattern.compile(
"(?i)(prof\\.\\s*|dr\\s*hab\\.\\s*|dr\\.\\s*|mgr\\s*|in\u017c\\.\\s*)",
Pattern.MULTILINE | Pattern.DOTALL
);
Matcher matcher = pattern.matcher(rawName);
return matcher.replaceAll("").trim();
}
}
@@ -0,0 +1,64 @@
package com.developx.szpitale.ingest.normalize;
import com.developx.szpitale.model.enums.AffiliationType;
import com.developx.szpitale.model.enums.ConfidenceLevel;
import java.util.Map;
import java.util.Set;
public class PartyNormalizer {
private static final Map<String, String> PARTY_CANONICAL_MAPPINGS = Map.ofEntries(
Map.entry("koalicja obywatelska", "Koalicja Obywatelska"),
Map.entry("po", "Koalicja Obywatelska"),
Map.entry("polska oferta", "Koalicja Obywatelska"),
Map.entry("prawo i sprawiedliwosc", "Prawo i Sprawiedliwo\u015b\u0107"),
Map.entry("pis", "Prawo i Sprawiedliwo\u015b\u0107"),
Map.entry("psl", "PSL"),
Map.entry("polska 2050", "Polska 2050"),
Map.entry("trzecia droga", "Trzecia Droga"),
Map.entry("polska 2050 pisl", "Trzecia Droga"),
Map.entry("sl d", "SLD"),
Map.entry("lewica", "Lewica"),
Map.entry("prawica rzeczpospolitej", "Prawica Rzeczypospolitej"),
Map.entry("ruch ludowy", "Ruch Ludowy"),
Map.entry("kww", "KWW_LOKALNY")
);
private static final Set<String> KOMMIT_SET = Set.of(
"kww", "komitet wyborczy", "komitet lokalny", "komitet wyborc\u00f3w"
);
public String canonicalize(String rawName) {
String lower = rawName.toLowerCase().trim();
if (PARTY_CANONICAL_MAPPINGS.containsKey(lower)) {
return PARTY_CANONICAL_MAPPINGS.get(lower);
}
for (Map.Entry<String, String> entry : PARTY_CANONICAL_MAPPINGS.entrySet()) {
if (lower.contains(entry.getKey())) {
return entry.getValue();
}
}
return rawName;
}
public AffiliationType determineType(String rawName) {
String lower = rawName.toLowerCase();
if (lower.contains("kww") || lower.contains("komitet wyborc\u00f3w")) {
return AffiliationType.KWW_LOKALNY;
}
if (lower.contains("komitet")) {
return AffiliationType.KOMITET_WYBORCZY;
}
return AffiliationType.PARTIA;
}
public ConfidenceLevel mapConfidence(String confidenceText) {
if (confidenceText == null) return ConfidenceLevel.BRAK_DANYCH;
String lower = confidenceText.toLowerCase();
if (lower.contains("brak danych")) return ConfidenceLevel.BRAK_DANYCH;
if (lower.contains("niezweryfik")) return ConfidenceLevel.NIEZWERYFIKOWANA;
if (lower.contains("niepotwierdzona")) return ConfidenceLevel.NIEPOTWIERDZONA;
return ConfidenceLevel.POTWIERDZONA;
}
}
@@ -0,0 +1,361 @@
package com.developx.szpitale.ingest.source;
import com.developx.szpitale.model.RawHospitalRecord;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
public class MarkdownRegistrySource implements VoivodeshipSource {
private static final Logger log = LoggerFactory.getLogger(MarkdownRegistrySource.class);
private final Path researchDir;
private final String voivodeshipSlug;
public MarkdownRegistrySource(Path researchDir, String voivodeshipSlug) {
this.researchDir = researchDir;
this.voivodeshipSlug = voivodeshipSlug;
}
@Override
public String voivodeship() {
return voivodeshipSlug;
}
@Override
public List<RawHospitalRecord> fetch() {
Path researchFile = findResearchFile(voivodeshipSlug);
if (researchFile == null) {
log.warn("No research file found for voivodeship: {}", voivodeshipSlug);
return List.of();
}
return parseMarkdownFile(researchFile);
}
private Path findResearchFile(String voivodeship) {
Path path = researchDir.resolve("research-" + voivodeship + ".md");
if (Files.exists(path)) {
return path;
}
path = researchDir.resolve("research-" + voivodeship + ".markdown");
if (Files.exists(path)) {
return path;
}
return null;
}
private List<RawHospitalRecord> parseMarkdownFile(Path filePath) {
List<RawHospitalRecord> records = new ArrayList<>();
try {
String content = Files.readString(filePath);
parseContent(content, records);
} catch (IOException e) {
log.error("Failed to read research file: {}", filePath, e);
}
return records;
}
private void parseContent(String content, List<RawHospitalRecord> records) {
String[] sections = content.split("## \\d+\\. ");
for (int i = 1; i < sections.length; i++) {
RawHospitalRecord record = parseHospitalSection(sections[i]);
if (record != null) {
records.add(record);
}
}
// Also try to parse with just "## " prefix (fallback)
if (records.isEmpty()) {
String[] fallbackSections = content.split("## ");
for (int i = 1; i < fallbackSections.length; i++) {
if (!fallbackSections[i].contains("Nota metodologiczna")
&& !fallbackSections[i].contains("Rozbiezności")
&& !fallbackSections[i].contains("Główne luki")
&& !fallbackSections[i].contains("Rekomendacje")) {
RawHospitalRecord record = parseHospitalSection(fallbackSections[i]);
if (record != null && record.hospitalName() != null && !record.hospitalName().contains("Nota")) {
records.add(record);
}
}
}
}
}
private RawHospitalRecord parseHospitalSection(String section) {
String[] lines = section.split("\n", 3);
if (lines.length < 2) return null;
String headerLine = lines[0].trim();
String hospitalName = headerLine.replace("*", "").trim();
if (hospitalName.isEmpty() || hospitalName.startsWith("|")) return null;
String body = lines.length > 2 ? lines[2] : "";
String legalForm = extractLegalForm(body);
String foundingBody = extractFoundingBody(body);
String supervisoryBodyType = extractSupervisoryBodyType(body);
List<RawHospitalRecord.RawPersonEntry> persons = parsePersonTables(section, legalForm, supervisoryBodyType,
extractWebsite(section), hospitalName);
boolean hasData = persons.stream().anyMatch(p -> p.sourceUrl() != null && !p.sourceUrl().isEmpty() && !p.sourceUrl().contains("brak"));
return new RawHospitalRecord(
hospitalName,
extractShortName(hospitalName),
extractCity(hospitalName),
voivodeship(),
legalForm,
foundingBody,
supervisoryBodyType,
null,
extractKrs(body),
extractWebsite(headerLine + "\n" + body),
persons
);
}
private List<RawHospitalRecord.RawPersonEntry> parsePersonTables(String section, String defaultLegalForm,
String defaultSupervisory, String defaultWebsite, String hospitalName) {
List<RawHospitalRecord.RawPersonEntry> entries = new ArrayList<>();
Document doc = Jsoup.parse(section);
Elements tables = doc.select("table");
for (Element table : tables) {
Elements rows = table.select("tr");
if (rows.size() < 2) continue;
Elements headerCells = rows.get(0).select("th, td");
boolean isPersonTable = headerCells.stream().anyMatch(h ->
h.text().contains("Funkcja") && h.text().contains("Imię"));
if (!isPersonTable) continue;
for (int i = 1; i < rows.size(); i++) {
Element row = rows.get(i);
Elements cells = row.select("td");
if (cells.isEmpty()) continue;
if (cells.get(0).text().contains("brak danych") ||
cells.get(0).text().contains("brak imiennego")) {
break;
}
String function = cleanText(cells.get(0).html());
String fullName = cleanText(cells.get(1).html());
String affiliation = cells.size() > 2 ? cleanText(cells.get(2).html()) : "brak/niepotwierdzona";
String sourceUrl = cells.size() > 3 ? extractUrl(cells.get(3).html()) : "";
if (fullName.isEmpty() || "brak".equalsIgnoreCase(fullName)) continue;
if (function.contains("brak") && fullName.contains("brak danych")) continue;
String organ = detectOrgan(function, defaultSupervisory, defaultLegalForm);
String confidence = mapConfidence(affiliation);
String status = mapStatus(function);
// Extract display title from fullName
String displayTitle = extractDisplayTitle(fullName, function);
entries.add(new RawHospitalRecord.RawPersonEntry(
function, fullName, displayTitle, affiliation,
confidence, sourceUrl, organ, status
));
}
}
if (entries.isEmpty()) {
// Fallback: try to parse single-line entries
parseFallbackEntries(section, entries, defaultSupervisory, defaultLegalForm);
}
return entries;
}
private void parseFallbackEntries(String section, List<RawHospitalRecord.RawPersonEntry> entries,
String defaultSupervisory, String defaultLegalForm) {
String[] lines = section.split("\n");
Pattern personPattern = Pattern.compile(
"^\\s*\\|\\s*(Dyrektor|Z-ca|p\\.o\\.|Rada|Naczelna|Główne)\\b[^|]*\\|\\s*([^|]+)\\|([^|]*)\\|([^|]*)\\s*$",
Pattern.MULTILINE
);
for (String line : lines) {
Matcher matcher = personPattern.matcher(line);
if (matcher.find()) {
String function = cleanText(matcher.group(1));
String fullName = cleanText(matcher.group(2)).trim();
String affiliation = cleanText(matcher.group(3)).trim();
String sourceUrl = cleanText(matcher.group(4)).trim();
if (!fullName.isEmpty()) {
entries.add(new RawHospitalRecord.RawPersonEntry(
function, fullName, null, affiliation,
mapConfidence(affiliation), sourceUrl,
detectOrgan(function, defaultSupervisory, defaultLegalForm), ""
));
}
}
}
}
private String detectOrgan(String function, String defaultSupervisory, String defaultLegalForm) {
String lower = function.toLowerCase();
if (lower.contains("dyrektor") && !lower.contains("z-ca") && !lower.contains("p.o.")) {
return "DYREKCJA";
}
if (lower.contains("z-ca") || lower.contains("p.o.") ||
lower.contains("główn") || lower.contains("naczel") ||
lower.contains("przel")) {
return "DYREKCJA";
}
if (lower.contains("rada")) {
if (defaultSupervisory != null && defaultSupervisory.contains("NADZORCZA")) {
return "RADA_NADZORCZA";
}
return "RADA_SPOLECZNA";
}
if (defaultSupervisory != null && defaultSupervisory.contains("NADZORCZA")) {
return "RADA_NADZORCZA";
}
return "RADA_SPOLECZNA";
}
private String extractLegalForm(String body) {
if (body.contains("sp. z o.o.") || body.contains("spółka")) {
return "SPOLKA_Z_OO";
}
if (body.contains("Psychiatryczny")) {
return "SP_PSYCHIATRYCZNY_ZOZ";
}
return "SPZOZ";
}
private String extractFoundingBody(String body) {
var mat = Pattern.compile("Organ tworzący:\\s*(.+?)\\.(?:\\s|$)", Pattern.MULTILINE | Pattern.DOTALL).matcher(body);
if (mat.find()) {
return cleanText(mat.group(1)).trim();
}
return null;
}
private String extractSupervisoryBodyType(String body) {
if (body.contains("Rada Nadzorcza") || body.contains("RADA NADZORCZA")) {
return "RADA_NADZORCZA";
}
return "RADA_SPOLECZNA";
}
private String extractKrs(String body) {
var mat = Pattern.compile("(?:KRS\\s*|krs\\s*:\\s*)(\\d{8,10})").matcher(body);
if (mat.find()) {
return mat.group(1);
}
return null;
}
private String extractWebsite(String text) {
var mat = Pattern.compile("(https?://[^\\s|]+)").matcher(text);
if (mat.find()) {
return mat.group(1);
}
return null;
}
private String extractShortName(String fullName) {
if (fullName.contains("Kliniczny") && fullName.contains("w ")) {
var mat = Pattern.compile("([A-Z]{2,4})\\b")
.matcher(fullName.split("Kliniczny")[0]);
if (mat.find()) return "USK";
}
return null;
}
private String extractCity(String hospitalName) {
var mat = Pattern.compile("w\\s+([^,.(]+)").matcher(hospitalName);
if (mat.find()) {
return cleanText(mat.group(1)).trim();
}
return null;
}
private String extractDisplayTitle(String fullName, String function) {
if (function.contains("Dyrektor Naczelny")) {
return "Dyrektor Naczelny";
}
if (function.contains("p.o.")) {
return "p.o. " + function.replace("p.o. ", "");
}
return null;
}
private String mapConfidence(String affiliation) {
if (affiliation == null) return "BRAK_DANYCH";
String lower = affiliation.toLowerCase();
if (lower.contains("brak danych")) return "BRAK_DANYCH";
if (lower.contains("niezweryfik") || lower.contains("NIEZWERYFIKOWANE")) return "NIEZWERYFIKOWANA";
if (lower.contains("brak/niepotwierdzona") || lower.contains("niepotwierdzona") ||
lower.contains("niepotwierdzone") || lower.contains("niepotwierdzony")) return "NIEPOTWIERDZONA";
if (lower.contains("NIEZWERYFIKOWANE (zbieżność") || lower.contains("nieweryfikow")) return "NIEZWERYFIKOWANA";
if (affiliation.contains("KO") || affiliation.contains("PO") ||
affiliation.contains("PiS") || affiliation.contains("PSL") ||
affiliation.contains("Trzecia Droga") || affiliation.contains("Polska 2050") ||
affiliation.contains("KWW ") || affiliation.contains("KOMITET") ||
affiliation.contains("Prawica") || affiliation.contains("SLD") ||
affiliation.contains("Lewica")) {
return "POTWIERDZONA";
}
return "NIEPOTWIERDZONA";
}
private String mapStatus(String function) {
String lower = function.toLowerCase();
if (lower.contains("p.o.") || lower.contains("pełniący")) return "PELNIACY_OBOWIAZKI";
if (lower.contains("elekt")) return "ELEKT";
if (lower.contains("były") || lower.contains("dawniej")) return "BYLY";
return "AKTUALNY";
}
private String cleanText(String html) {
if (html == null) return "";
Document doc = Jsoup.parse(html);
String text = doc.text().trim();
// Remove trailing pipe if any
if (text.endsWith("|")) text = text.substring(0, text.length() - 1).trim();
return text;
}
private String extractUrl(String text) {
String cleaned = cleanText(text);
var mat = Pattern.compile("(https?://[^\\s]+)").matcher(cleaned);
if (mat.find()) return mat.group(1).replaceAll("[)]+$", "");
return cleaned.isEmpty() ? "" : cleaned;
}
public static List<VoivodeshipSource> discoverAllSources(Path researchDir) {
List<VoivodeshipSource> sources = new ArrayList<>();
try (var paths = Files.list(researchDir)) {
paths.filter(p -> p.toString().endsWith(".md"))
.filter(p -> p.getFileName().toString().startsWith("research-"))
.forEach(p -> {
String voiv = p.getFileName().toString().replace("research-", "").replace(".md", "");
sources.add(new MarkdownRegistrySource(researchDir, voiv));
log.info("Discovered research source for voivodeship: {}", voiv);
});
} catch (IOException e) {
log.warn("Could not scan research directory: {}", researchDir);
}
return sources;
}
}
@@ -0,0 +1,10 @@
package com.developx.szpitale.ingest.source;
import com.developx.szpitale.model.RawHospitalRecord;
import java.util.List;
public interface VoivodeshipSource {
String voivodeship();
List<RawHospitalRecord> fetch();
}
@@ -0,0 +1,53 @@
package com.developx.szpitale.load;
import com.developx.szpitale.model.CanonicalDataset;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
public class CanonicalReader {
private final ObjectMapper objectMapper = new ObjectMapper();
public CanonicalReader() {
objectMapper.registerModule(new JavaTimeModule());
}
public List<CanonicalDataset> readAll(Path inputDir, String voivodeship) throws IOException {
List<CanonicalDataset> datasets = new ArrayList<>();
Path canonicalDir = inputDir.resolve("canonical").toAbsolutePath().normalize();
if (!Files.exists(canonicalDir)) {
System.err.println("Error: canonical directory not found: " + canonicalDir);
return datasets;
}
try (var paths = Files.list(canonicalDir)) {
paths.filter(p -> p.getFileName().toString().endsWith(".json"))
.filter(p -> {
if ("all".equalsIgnoreCase(voivodeship)) return true;
String base = p.getFileName().toString().replace(".json", "");
return base.equalsIgnoreCase(voivodeship);
})
.forEach(p -> {
try {
CanonicalDataset ds = objectMapper.readValue(p.toFile(), CanonicalDataset.class);
datasets.add(ds);
} catch (IOException e) {
System.err.println("Error reading " + p + ": " + e.getMessage());
}
});
}
return datasets;
}
public CanonicalDataset readSingle(Path inputDir, String voivodeship) throws IOException {
return readAll(inputDir, voivodeship).stream().findFirst().orElse(null);
}
}
@@ -0,0 +1,216 @@
package com.developx.szpitale.load;
import com.developx.szpitale.model.*;
import org.neo4j.driver.Driver;
import org.neo4j.driver.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.stream.Collectors;
public class GraphLoader {
private static final Logger log = LoggerFactory.getLogger(GraphLoader.class);
private static final int BATCH_SIZE = 1000;
private final Driver driver;
public GraphLoader(Driver driver) {
this.driver = driver;
}
public LoadStats load(List<CanonicalDataset> datasets) {
LoadStats stats = new LoadStats();
try (Session session = driver.session()) {
for (CanonicalDataset dataset : datasets) {
stats.addVoivodeship(dataset.voivodeship());
loadHospitals(session, dataset.hospitals(), stats);
loadPeople(session, dataset.people(), stats);
loadRoles(session, dataset.roles(), stats);
loadAffiliations(session, dataset.affiliations(), stats);
loadMandates(session, dataset.mandates(), stats);
}
}
return stats;
}
private void loadHospitals(Session session, List<Hospital> hospitals, LoadStats stats) {
List<Map<String, Object>> batch = new ArrayList<>();
for (Hospital h : hospitals) {
batch.add(Map.of(
"id", h.id(),
"name", h.name(),
"shortName", h.shortName(),
"city", h.city(),
"voivodeship", h.voivodeship(),
"legalForm", h.legalForm() != null ? h.legalForm().name() : null,
"foundingBody", h.foundingBody(),
"supervisoryBodyType", h.supervisoryBodyType() != null ? h.supervisoryBodyType().name() : null,
"website", h.website(),
"sourceUrls", h.sourceUrls()
));
}
for (int i = 0; i < batch.size(); i += BATCH_SIZE) {
List<Map<String, Object>> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size()));
String cypher = "UNWIND $batch AS h " +
"MERGE (hospital:Hospital {id: h.id}) " +
"SET hospital += {name:h.name, shortName:h.shortName, city:h.city, " +
" voivodeship:h.voivodeship, legalForm:h.legalForm, " +
" foundingBody:h.foundingBody, supervisoryBodyType:h.supervisoryBodyType, " +
" website:h.website} " +
"MERGE (v:Voivodeship {slug: h.voivodeship}) " +
"MERGE (hospital)-[:W_WOJEWODZTWIE]->(v)";
session.run(cypher, Map.of("batch", subList));
stats.addHospitals(subList.size());
}
}
private void loadPeople(Session session, List<Person> people, LoadStats stats) {
List<Map<String, Object>> batch = new ArrayList<>();
for (Person p : people) {
batch.add(Map.of(
"id", p.id(),
"fullName", p.fullName(),
"displayName", p.displayName(),
"titles", p.titles() != null ? p.titles() : Collections.emptyList(),
"sourceUrls", p.sourceUrls()
));
}
for (int i = 0; i < batch.size(); i += BATCH_SIZE) {
List<Map<String, Object>> subList = batch.subList(i, Math.min(i + BATCH_SIZE, batch.size()));
String cypher = "UNWIND $batch AS p " +
"MERGE (person:Person {id: p.id}) " +
"SET person += {fullName:p.fullName, displayName:p.displayName, " +
" titles:p.titles, sourceUrls:p.sourceUrls}";
session.run(cypher, Map.of("batch", subList));
stats.addPeople(subList.size());
}
}
private void loadRoles(Session session, List<Role> roles, LoadStats stats) {
Map<String, List<Role>> byRelType = new LinkedHashMap<>();
for (Role r : roles) {
String relType = roleTypeToRelName(r);
byRelType.computeIfAbsent(relType, k -> new ArrayList<>()).add(r);
}
for (Map.Entry<String, List<Role>> entry : byRelType.entrySet()) {
String relType = entry.getKey();
List<Role> items = entry.getValue();
for (int i = 0; i < items.size(); i += BATCH_SIZE) {
List<Role> subList = items.subList(i, Math.min(i + BATCH_SIZE, items.size()));
String cypher = "UNWIND $batch AS r " +
"MATCH (h:Hospital {id: r.hospitalId}) " +
"MATCH (p:Person {id: r.personId}) " +
"MERGE (h)-[rrel:`" + escapeBacktick(relType) + "`]->(p) " +
"SET rrel += {roleType: r.roleType, roleLabel: r.roleLabel, " +
" organ: r.organ, status: r.status, " +
" validFrom: r.validFrom, validTo: r.validTo, note: r.note}";
session.run(cypher, Map.of("batch", subList.stream()
.map(r -> Map.of(
"hospitalId", r.hospitalId(),
"personId", r.personId(),
"roleType", r.roleType().name(),
"roleLabel", r.roleLabel(),
"organ", r.organ().name(),
"status", r.status().name(),
"validFrom", r.validFrom(),
"validTo", r.validTo(),
"note", r.note()
))
.collect(Collectors.toList())));
stats.addRoles(subList.size());
}
}
}
private void loadAffiliations(Session session, List<Affiliation> affiliations, LoadStats stats) {
for (int i = 0; i < affiliations.size(); i += BATCH_SIZE) {
List<Affiliation> subList = affiliations.subList(
i, Math.min(i + BATCH_SIZE, affiliations.size()));
String cypher = "UNWIND $batch AS a " +
"MATCH (p:Person {id: a.personId}) " +
"MERGE (party:Party {name: a.name}) SET party.type = a.type " +
"MERGE (p)-[m:CZLONEK_PARTII]->(party) " +
"SET m += {confidence: a.confidence, note: a.note, sourceUrls: a.sourceUrls}";
session.run(cypher, Map.of("batch", subList.stream()
.map(a -> Map.of(
"personId", a.personId(),
"name", a.name(),
"type", a.type().name(),
"confidence", a.confidence().name(),
"note", a.note(),
"sourceUrls", a.sourceUrls()
))
.collect(Collectors.toList())));
stats.addAffiliations(subList.size());
}
}
private void loadMandates(Session session, List<Mandate> mandates, LoadStats stats) {
if (mandates.isEmpty()) return;
for (int i = 0; i < mandates.size(); i += BATCH_SIZE) {
List<Mandate> subList = mandates.subList(
i, Math.min(i + BATCH_SIZE, mandates.size()));
String cypher = "UNWIND $batch AS m " +
"MATCH (p:Person {id: m.personId}) " +
"MERGE (g:GovBody {name: m.body}) " +
" SET g.type = m.mandateType, g.voivodeship = m.voivodeship, g.term = m.term " +
"MERGE (p)-[:PELNI_MANDAT {mandateType: m.mandateType, term: m.term}]->(g)";
session.run(cypher, Map.of("batch", subList.stream()
.map(m -> Map.of(
"personId", m.personId(),
"body", m.body(),
"mandateType", m.mandateType().name(),
"term", m.term(),
"voivodeship", m.voivodeship(),
"sourceUrls", m.sourceUrls()
))
.collect(Collectors.toList())));
stats.addMandates(subList.size());
}
}
private String roleTypeToRelName(Role r) {
String organ = r.organ().name();
String role = r.roleType().name();
if ("DYREKCJA".equals(organ) && "DYREKTOR".equals(role)) return "DYREKTOR";
if ("DYREKCJA".equals(organ) && ("ZASTEPCA_DYREKTORA".equals(role) || "GLOWNY_KSIEGOWY".equals(role) || "PRZELOZONA_PIELEGNIARKOW".equals(role))) return "ZASTEPCA_DYREKTORA";
if ("PRZEWODNICZACY_ORGANU".equals(role)) return "PRZEWODNICZY_ORGANOWI";
return "CZLONEK_ORGANU";
}
private String escapeBacktick(String s) {
return s.replace("`", "\\`");
}
public static class LoadStats {
public int hospitalCount;
public int personCount;
public int roleCount;
public int affiliationCount;
public int mandateCount;
public final List<String> voivodeships = new ArrayList<>();
public void addVoivodeship(String v) { voivodeships.add(v); }
public void addHospitals(int count) { hospitalCount += count; }
public void addPeople(int count) { personCount += count; }
public void addRoles(int count) { roleCount += count; }
public void addAffiliations(int count) { affiliationCount += count; }
public void addMandates(int count) { mandateCount += count; }
@Override
public String toString() {
return String.format(
"Load stats - voivodeships: %s, hospitals: %d, people: %d, roles: %d, affiliations: %d, mandates: %d",
String.join(", ", voivodeships), hospitalCount, personCount, roleCount, affiliationCount, mandateCount
);
}
}
}
@@ -0,0 +1,36 @@
package com.developx.szpitale.load;
import org.neo4j.driver.Driver;
import org.neo4j.driver.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class GraphSchema {
private static final Logger log = LoggerFactory.getLogger(GraphSchema.class);
private final Driver driver;
public GraphSchema(Driver driver) {
this.driver = driver;
}
public void createConstraints() {
try (Session session = driver.session()) {
session.run("CREATE CONSTRAINT hospital_id IF NOT EXISTS FOR (h:Hospital) REQUIRE h.id IS UNIQUE");
session.run("CREATE CONSTRAINT person_id IF NOT EXISTS FOR (p:Person) REQUIRE p.id IS UNIQUE");
session.run("CREATE CONSTRAINT party_name IF NOT EXISTS FOR (x:Party) REQUIRE x.name IS UNIQUE");
session.run("CREATE CONSTRAINT voiv_slug IF NOT EXISTS FOR (v:Voivodeship) REQUIRE v.slug IS UNIQUE");
session.run("CREATE INDEX person_name IF NOT EXISTS FOR (p:Person) ON (p.fullName)");
session.run("CREATE INDEX govbody_name IF NOT EXISTS FOR (g:GovBody) ON (g.name)");
log.info("Graph constraints and indexes created");
}
}
public void dropAll() {
try (Session session = driver.session()) {
session.run("MATCH (n) DETACH DELETE n");
log.info("All nodes and relationships deleted");
}
}
}
@@ -0,0 +1,126 @@
package com.developx.szpitale.load;
import org.neo4j.driver.Driver;
import org.neo4j.driver.Session;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
public class GraphValidator {
private static final Logger log = LoggerFactory.getLogger(GraphValidator.class);
private final Driver driver;
public GraphValidator(Driver driver) {
this.driver = driver;
}
public ValidationReport validate() {
ValidationReport report = new ValidationReport();
try (Session session = driver.session()) {
// 1. Hospitals without directors
var result1 = session.run(
"MATCH (h:Hospital) WHERE NOT (h)-[:DYREKTOR]->() " +
"RETURN h.name AS hospital, h.city AS city, h.voivodeship AS voivodeship " +
"ORDER BY h.voivodeship, h.name");
List<Map<String, Object>> hospitalsWithoutDirector = new ArrayList<>();
result1.list().forEach(r -> hospitalsWithoutDirector.add(r.asMap()));
report.setHospitalsWithoutDirector(hospitalsWithoutDirector);
// 2. People with mandates linking to hospital roles
var result2 = session.run(
"MATCH (h:Hospital)-[r:CZLONEK_ORGANU|PRZEWODNICZY_ORGANOWI]->(p:Person)-[:PELNI_MANDAT]->(g:GovBody) " +
"RETURN p.fullName AS person, " +
" collect(DISTINCT h.name) AS szpitale, " +
" collect(DISTINCT g.name) AS mandaty, " +
" collect(DISTINCT labels(p)) AS labels " +
"ORDER BY person");
List<Map<String, Object>> overlaps = new ArrayList<>();
result2.list().forEach(r -> overlaps.add(r.asMap()));
report.setOverlapReport(overlaps);
// 3. People in multiple hospitals
var result3 = session.run(
"MATCH (h:Hospital)-[r:CZLONEK_ORGANU|PRZEWODNICZY_ORGANOWI]->(p:Person) " +
"WITH p, collect(DISTINCT h.name) AS szpitale WHERE size(szpitale) > 1 " +
"RETURN p.fullName AS person, szpitale, count(p) AS connections " +
"ORDER BY connections DESC");
List<Map<String, Object>> multiHospital = new ArrayList<>();
result3.list().forEach(r -> multiHospital.add(r.asMap()));
report.setMultiHospitalReport(multiHospital);
// 4. Overall counts
var nodeCountResult = session.run(
"MATCH (n) RETURN labels(n)[0] AS label, count(*) AS count ORDER BY count DESC");
Map<String, Long> nodeCounts = new HashMap<>();
nodeCountResult.list().forEach(r ->
nodeCounts.put(r.get("label").asString(), r.get("count").asLong()));
report.setNodeCounts(nodeCounts);
var relCountResult = session.run(
"MATCH ()-[rel]->() RETURN type(rel) AS type, count(*) AS count ORDER BY count DESC");
Map<String, Long> relCounts = new HashMap<>();
relCountResult.list().forEach(r ->
relCounts.put(r.get("type").asString(), r.get("count").asLong()));
report.setRelCounts(relCounts);
var totalNodes = session.run("MATCH (n) RETURN count(*) AS total").single().get("total").asLong();
report.setTotalNodes(totalNodes);
var totalRels = session.run("MATCH ()-[rel]->() RETURN count(*) AS total").single().get("total").asLong();
report.setTotalRelationships(totalRels);
}
return report;
}
public void exportOverlapCsv(Path outputPath, List<Map<String, Object>> overlaps) throws IOException {
StringBuilder csv = new StringBuilder();
csv.append("osoba;szpitale;mandaty;afiliacje;confidence;sourceUrls\n");
for (Map<String, Object> row : overlaps) {
String person = (String) row.getOrDefault("person", "");
List<?> szpitale = (List<?>) row.getOrDefault("szpitale", List.of());
List<?> mandaty = (List<?>) row.getOrDefault("mandaty", List.of());
csv.append(person)
.append(";").append(String.join(", ", szpitale.stream().map(Object::toString).toList()))
.append(";").append(String.join(", ", mandaty.stream().map(Object::toString).toList()))
.append("\n");
}
Files.writeString(outputPath, csv.toString());
log.info("Overlap report written to: {}", outputPath);
}
public static class ValidationReport {
public List<Map<String, Object>> hospitalsWithoutDirector = new ArrayList<>();
public List<Map<String, Object>> overlapReport = new ArrayList<>();
public List<Map<String, Object>> multiHospitalReport = new ArrayList<>();
public Map<String, Long> nodeCounts = new HashMap<>();
public Map<String, Long> relCounts = new HashMap<>();
public long totalNodes;
public long totalRelationships;
public void setHospitalsWithoutDirector(List<Map<String, Object>> list) { this.hospitalsWithoutDirector = list; }
public void setOverlapReport(List<Map<String, Object>> list) { this.overlapReport = list; }
public void setMultiHospitalReport(List<Map<String, Object>> list) { this.multiHospitalReport = list; }
public void setNodeCounts(Map<String, Long> map) { this.nodeCounts = map; }
public void setRelCounts(Map<String, Long> map) { this.relCounts = map; }
public void setTotalNodes(long n) { this.totalNodes = n; }
public void setTotalRelationships(long r) { this.totalRelationships = r; }
@Override
public String toString() {
return "Validation report:\n" +
" Total nodes: " + totalNodes + "\n" +
" Total relationships: " + totalRelationships + "\n" +
" Node types: " + nodeCounts + "\n" +
" Relationship types: " + relCounts + "\n" +
" Hospitals without directors: " + hospitalsWithoutDirector.size() + "\n" +
" People with mandates: " + overlapReport.size() + "\n" +
" People in multiple hospitals: " + multiHospitalReport.size() + "\n";
}
}
}
@@ -0,0 +1,54 @@
package com.developx.szpitale.load;
import org.neo4j.driver.Driver;
import java.io.IOException;
import java.nio.file.Path;
public class LoadCommand {
private final Driver neo4jDriver;
public LoadCommand(Driver neo4jDriver) {
this.neo4jDriver = neo4jDriver;
}
public String load(String inDir, String voivodeship, boolean createSchema, boolean wipe, boolean validate) {
try {
if (wipe) {
new GraphSchema(neo4jDriver).dropAll();
System.out.println("Graph wiped.");
}
if (createSchema) {
new GraphSchema(neo4jDriver).createConstraints();
System.out.println("Schema created.");
}
CanonicalReader reader = new CanonicalReader();
var datasets = reader.readAll(Path.of(inDir), voivodeship);
if (datasets.isEmpty()) {
return "Error: No canonical datasets found for voivodeship: " + voivodeship;
}
GraphLoader loader = new GraphLoader(neo4jDriver);
GraphLoader.LoadStats stats = loader.load(datasets);
System.out.println(stats);
if (validate) {
GraphValidator validator = new GraphValidator(neo4jDriver);
GraphValidator.ValidationReport report = validator.validate();
System.out.println(report);
try {
validator.exportOverlapCsv(Path.of("overlap_report.csv"), report.overlapReport);
} catch (IOException e) {
System.err.println("Error writing overlap report: " + e.getMessage());
}
}
return "Load completed successfully.";
} catch (Exception e) {
return "Error during load: " + e.getMessage();
}
}
}
@@ -0,0 +1,17 @@
package com.developx.szpitale.model;
import com.developx.szpitale.model.enums.AffiliationType;
import com.developx.szpitale.model.enums.ConfidenceLevel;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
public record Affiliation(
@JsonProperty("personId") String personId,
@JsonProperty("type") AffiliationType type,
@JsonProperty("name") String name,
@JsonProperty("confidence") ConfidenceLevel confidence,
@JsonProperty("note") String note,
@JsonProperty("sourceUrls") List<String> sourceUrls
) {
}
@@ -0,0 +1,18 @@
package com.developx.szpitale.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.LocalDateTime;
import java.util.List;
public record CanonicalDataset(
@JsonProperty("schemaVersion") String schemaVersion,
@JsonProperty("voivodeship") String voivodeship,
@JsonProperty("generatedAt") LocalDateTime generatedAt,
@JsonProperty("hospitals") List<Hospital> hospitals,
@JsonProperty("people") List<Person> people,
@JsonProperty("affiliations") List<Affiliation> affiliations,
@JsonProperty("roles") List<Role> roles,
@JsonProperty("mandates") List<Mandate> mandates
) {
}
@@ -0,0 +1,23 @@
package com.developx.szpitale.model;
import com.developx.szpitale.model.enums.LegalForm;
import com.developx.szpitale.model.enums.SupervisoryBodyType;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
public record Hospital(
@JsonProperty("id") String id,
@JsonProperty("name") String name,
@JsonProperty("shortName") String shortName,
@JsonProperty("city") String city,
@JsonProperty("voivodeship") String voivodeship,
@JsonProperty("legalForm") LegalForm legalForm,
@JsonProperty("foundingBody") String foundingBody,
@JsonProperty("supervisoryBodyType") SupervisoryBodyType supervisoryBodyType,
@JsonProperty("nip") String nip,
@JsonProperty("krs") String krs,
@JsonProperty("website") String website,
@JsonProperty("sourceUrls") List<String> sourceUrls
) {
}
@@ -0,0 +1,16 @@
package com.developx.szpitale.model;
import com.developx.szpitale.model.enums.MandateType;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
public record Mandate(
@JsonProperty("personId") String personId,
@JsonProperty("mandateType") MandateType mandateType,
@JsonProperty("body") String body,
@JsonProperty("term") String term,
@JsonProperty("voivodeship") String voivodeship,
@JsonProperty("sourceUrls") List<String> sourceUrls
) {
}
@@ -0,0 +1,14 @@
package com.developx.szpitale.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;
public record Person(
@JsonProperty("id") String id,
@JsonProperty("fullName") String fullName,
@JsonProperty("displayName") String displayName,
@JsonProperty("titles") List<String> titles,
@JsonProperty("sourceUrls") List<String> sourceUrls
) {
}
@@ -0,0 +1,41 @@
package com.developx.szpitale.model;
import java.util.List;
import java.util.Map;
public record RawHospitalRecord(
String hospitalName,
String shortName,
String city,
String voivodeship,
String legalForm,
String foundingBody,
String supervisoryBodyType,
String nip,
String krs,
String website,
List<RawPersonEntry> persons
) {
public record RawPersonEntry(
String function,
String fullName,
String displayTitle,
String partyAffiliation,
String confidence,
String sourceUrl,
String organ,
String statusNote
) {
}
public record RawMandate(
String personName,
String mandateType,
String body,
String term,
String voivodeship,
String sourceUrl
) {
}
}
@@ -0,0 +1,25 @@
package com.developx.szpitale.model;
import com.developx.szpitale.model.enums.OrganType;
import com.developx.szpitale.model.enums.RoleStatus;
import com.developx.szpitale.model.enums.RoleType;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.LocalDate;
import java.util.List;
public record Role(
@JsonProperty("hospitalId") String hospitalId,
@JsonProperty("personId") String personId,
@JsonProperty("roleType") RoleType roleType,
@JsonProperty("roleLabel") String roleLabel,
@JsonProperty("organ") OrganType organ,
@JsonProperty("status") RoleStatus status,
@JsonProperty("validFrom") LocalDate validFrom,
@JsonProperty("validTo") LocalDate validTo,
@JsonProperty("note") String note,
@JsonProperty("sourceUrls") List<String> sourceUrls
) {
}
@@ -0,0 +1,7 @@
package com.developx.szpitale.model.enums;
public enum AffiliationType {
PARTIA,
KOMITET_WYBORCZY,
KWW_LOKALNY
}
@@ -0,0 +1,8 @@
package com.developx.szpitale.model.enums;
public enum ConfidenceLevel {
POTWIERDZONA,
NIEPOTWIERDZONA,
NIEZWERYFIKOWANA,
BRAK_DANYCH
}
@@ -0,0 +1,8 @@
package com.developx.szpitale.model.enums;
public enum LegalForm {
SPZOZ,
SP_PSYCHIATRYCZNY_ZOZ,
SPOLKA_Z_OO,
INNY
}
@@ -0,0 +1,12 @@
package com.developx.szpitale.model.enums;
public enum MandateType {
RADNY_SEJMIKU,
RADNY_POWIATU,
RADNY_GMINY,
WOJT_BURMISTRZ_PREZYDENT,
STAROSTA,
POSEL,
SENATOR,
MARSZALEK
}
@@ -0,0 +1,7 @@
package com.developx.szpitale.model.enums;
public enum OrganType {
DYREKCJA,
RADA_SPOLECZNA,
RADA_NADZORCZA
}
@@ -0,0 +1,8 @@
package com.developx.szpitale.model.enums;
public enum RoleStatus {
AKTUALNY,
PELNIACY_OBOWIAZKI,
ELEKT,
BYLY
}
@@ -0,0 +1,12 @@
package com.developx.szpitale.model.enums;
public enum RoleType {
DYREKTOR,
ZASTEPCA_DYREKTORA,
GLOWNY_KSIEGOWY,
PRZELOZONA_PIELEGNIARKOW,
CZLONEK_ORGANU_NADZORCZEGO,
PRZEWODNICZACY_ORGANU,
SEKRETARZ_ORGANU,
Z_CA_PRZEWODNICZACEGO
}
@@ -0,0 +1,6 @@
package com.developx.szpitale.model.enums;
public enum SupervisoryBodyType {
RADA_SPOLECZNA,
RADA_NADZORCZA
}
@@ -0,0 +1,14 @@
spring:
neo4j:
uri: bolt://localhost:7687
authentication:
username: neo4j
password: password
application:
name: szpitale-graph
app:
ingest:
canonical-dir: canonical
neo4j:
batch-size: 1000
@@ -0,0 +1,68 @@
package com.developx.szpitale.ingest.normalize;
import com.developx.szpitale.model.Person;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
class DeduplicatorTest {
private Deduplicator deduplicator;
@BeforeEach
void setUp() {
deduplicator = new Deduplicator();
}
@Test
void checkFuzzyDuplicates_similarNames_reportedNotMerged() {
Person janKowalski = new Person(
"person:jan-kowalski",
"Jan Kowalski",
"Jan Kowalski",
List.of(),
List.of("https://example.com/1")
);
Person janKowaski = new Person(
"person:jan-kowaski",
"Jan Kowaski",
"Jan Kowaski",
List.of(),
List.of("https://example.com/2")
);
List<Person> input = List.of(janKowalski, janKowaski);
List<Person> result = deduplicator.deduplicate(input);
assertEquals(2, result.size(), "Should not merge two similar names - both should remain as separate entries");
assertTrue(result.stream().anyMatch(p -> p.id().equals("person:jan-kowalski")));
assertTrue(result.stream().anyMatch(p -> p.id().equals("person:jan-kowaski")));
}
@Test
void deduplicate_exactSlugMatches_merged() {
Person first = new Person(
"person:jan-kowalski",
"Jan Kowalski",
"Jan Kowalski",
List.of(),
List.of("https://example.com/1")
);
Person second = new Person(
"person:jan-kowalski",
"Jan Kowalski",
"Jan Kowalski",
List.of(),
List.of("https://example.com/2")
);
List<Person> result = deduplicator.deduplicate(List.of(first, second));
assertEquals(1, result.size(), "Should merge persons with same slug");
assertEquals(2, result.get(0).sourceUrls().size(), "Merged person should have all source URLs");
}
}
@@ -0,0 +1,14 @@
spring:
neo4j:
uri: bolt://localhost:7687
authentication:
username: neo4j
password: password
application:
name: szpitale-graph
app:
ingest:
canonical-dir: canonical
neo4j:
batch-size: 1000
@@ -0,0 +1,26 @@
com/developx/szpitale/ingest/IngestResults.class
com/developx/szpitale/load/CanonicalReader.class
com/developx/szpitale/ingest/source/VoivodeshipSource.class
com/developx/szpitale/model/enums/AffiliationType.class
com/developx/szpitale/ingest/source/MarkdownRegistrySource.class
com/developx/szpitale/SzpitaleGraphApplication.class
com/developx/szpitale/load/GraphValidator.class
com/developx/szpitale/ingest/IngestCommand.class
com/developx/szpitale/ingest/normalize/PartyNormalizer.class
com/developx/szpitale/load/LoadCommand.class
com/developx/szpitale/load/GraphValidator$ValidationReport.class
com/developx/szpitale/ingest/normalize/Deduplicator.class
com/developx/szpitale/load/GraphSchema.class
com/developx/szpitale/model/enums/OrganType.class
com/developx/szpitale/model/RawHospitalRecord$RawMandate.class
com/developx/szpitale/model/enums/ConfidenceLevel.class
com/developx/szpitale/ingest/normalize/NameNormalizer.class
com/developx/szpitale/model/RawHospitalRecord$RawPersonEntry.class
com/developx/szpitale/model/enums/SupervisoryBodyType.class
com/developx/szpitale/load/GraphLoader.class
com/developx/szpitale/model/RawHospitalRecord.class
com/developx/szpitale/ingest/CanonicalWriter.class
com/developx/szpitale/load/GraphLoader$LoadStats.class
com/developx/szpitale/model/enums/RoleType.class
com/developx/szpitale/model/enums/LegalForm.class
com/developx/szpitale/model/enums/RoleStatus.class
@@ -0,0 +1,29 @@
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/SzpitaleGraphApplication.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/CanonicalWriter.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/IngestCommand.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/IngestResults.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/normalize/Deduplicator.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/normalize/NameNormalizer.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/normalize/PartyNormalizer.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/source/MarkdownRegistrySource.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/ingest/source/VoivodeshipSource.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/load/CanonicalReader.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/load/GraphLoader.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/load/GraphSchema.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/load/GraphValidator.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/load/LoadCommand.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/Affiliation.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/CanonicalDataset.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/Hospital.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/Mandate.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/Person.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/RawHospitalRecord.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/Role.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/AffiliationType.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/ConfidenceLevel.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/LegalForm.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/MandateType.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/OrganType.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/RoleStatus.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/RoleType.java
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/main/java/com/developx/szpitale/model/enums/SupervisoryBodyType.java
@@ -0,0 +1 @@
com/developx/szpitale/ingest/normalize/DeduplicatorTest.class
@@ -0,0 +1 @@
/home/kruszewskia/Workspace/Private/Szpitale-graph/szpitale-graph/src/test/java/com/developx/szpitale/ingest/normalize/DeduplicatorTest.java
File diff suppressed because one or more lines are too long
@@ -0,0 +1,4 @@
-------------------------------------------------------------------------------
Test set: com.developx.szpitale.ingest.normalize.DeduplicatorTest
-------------------------------------------------------------------------------
Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.028 s -- in com.developx.szpitale.ingest.normalize.DeduplicatorTest