kempersc/glam
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(scripts): force table recreation in web archives migration

Browse source 
Drop existing tables before creating to ensure schema updates are applied
properly instead of using IF NOT EXISTS which would skip schema changes.
This commit is contained in:
kempersc 2025-12-07 18:47:46 +01:00
parent 0c4c378e06
commit 400b1c04c1
1 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
scripts/migrate_web_archives.py
View file

@ -168,9 +168,14 @@ def build_ducklake_database(mapping: Dict[int, str]):
con = duckdb.connect(str(DUCKLAKE_DB))
# Drop and recreate tables to ensure schema is up to date
con.execute("DROP TABLE IF EXISTS web_claims")
con.execute("DROP TABLE IF EXISTS web_pages")
con.execute("DROP TABLE IF EXISTS web_archives")
# Create tables
con.execute("""
CREATE TABLE IF NOT EXISTS web_archives (
CREATE TABLE web_archives (
ghcid VARCHAR PRIMARY KEY,
entry_index INTEGER,
domain VARCHAR,
@ -186,7 +191,7 @@ def build_ducklake_database(mapping: Dict[int, str]):
""")
con.execute("""
CREATE TABLE IF NOT EXISTS web_pages (
CREATE TABLE web_pages (
id INTEGER PRIMARY KEY,
ghcid VARCHAR,
page_title VARCHAR,
@ -198,7 +203,7 @@ def build_ducklake_database(mapping: Dict[int, str]):
""")
con.execute("""
CREATE TABLE IF NOT EXISTS web_claims (
CREATE TABLE web_claims (
id INTEGER PRIMARY KEY,
ghcid VARCHAR,
claim_id VARCHAR,
@ -217,9 +222,9 @@ def build_ducklake_database(mapping: Dict[int, str]):
""")
# Clear existing data
con.execute("DELETE FROM web_claims")
con.execute("DELETE FROM web_pages")
con.execute("DELETE FROM web_archives")
con.execute("-- Removed: DELETE FROM web_claims")
con.execute("-- Removed: DELETE FROM web_pages")
con.execute("-- Removed: DELETE FROM web_archives")
page_id = 0
claim_id_counter = 0