assive database create tables buildout

postgresql/migrations/v1_11-05-23_migrations_script.py

@@ -0,0 +1,96 @@
+
+import psycopg2
+import os
+import logging
+from contextlib import contextmanager
+
+# Configure logging
+logging.basicConfig(
+    filename='migration.log',
+    filemode='a',
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    level=logging.INFO
+)
+
+# Database connection parameters
+db_params = {
+    "dbname": os.getenv("DB_NAME"),
+    "user": os.getenv("DB_USER"),
+    "password": os.getenv("DB_PASSWORD"),
+    "host": os.getenv("DB_HOST"),
+    "port": os.getenv("DB_PORT")
+}
+
+# Context manager for database connection
+@contextmanager
+def get_db_connection():
+    conn = psycopg2.connect(**db_params)
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+# Context manager for database cursor
+@contextmanager
+def get_db_cursor(commit=False):
+    with get_db_connection() as conn:
+        cursor = conn.cursor()
+        try:
+            yield cursor
+            if commit:
+                conn.commit()
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Transaction failed: {e}")
+            raise
+        finally:
+            cursor.close()
+
+# Function to check if a migration was applied
+def is_migration_applied(cursor, migration_name):
+    cursor.execute("SELECT COUNT(*) FROM applied_migrations WHERE migration_name = %s", (migration_name,))
+    return cursor.fetchone()[0] > 0
+
+# Function to record a migration as applied
+def record_migration(cursor, migration_name):
+    cursor.execute("INSERT INTO applied_migrations (migration_name) VALUES (%s)", (migration_name,))
+
+# Function to apply a migration script
+def apply_migration(cursor, filepath):
+    filename = os.path.basename(filepath)
+    if is_migration_applied(cursor, filename):
+        logging.info(f"Migration {filename} already applied.")
+        return
+
+    logging.info(f"Applying migration {filename}")
+    with open(filepath, 'r') as file:
+        cursor.execute(file.read())
+    record_migration(cursor, filename)
+    logging.info(f"Migration {filename} applied successfully.")
+
+# Main migration function
+def migrate():
+    # List of migration scripts
+    migration_scripts = ['path_to_your_first_migration_script.sql', 'path_to_your_next_migration_script.sql']  # etc.
+
+    with get_db_cursor(commit=True) as cursor:
+        # Ensure the migrations tracking table exists
+        cursor.execute("""
+            CREATE TABLE IF NOT EXISTS applied_migrations (
+                migration_name TEXT PRIMARY KEY,
+                applied_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+            );
+        """)
+
+        # Apply each migration script
+        for script_path in migration_scripts:
+            apply_migration(cursor, script_path)
+
+def main():
+    try:
+        migrate()
+    except Exception as e:
+        logging.error(f"Migration failed: {e}")
+
+if __name__ == '__main__':
+    main()

postgresql/schemas/01_init_db.sh

@@ -1,44 +1,46 @@
 #!/bin/bash
 
-# This script initializes the PostgreSQL environment for your project.
 # Replace 'versus_rex' with your desired database name,
 # 'oedypus' with your desired username, and 'Zarathustra22!' with your desired password.
 
 # Exit immediately if a command exits with a non-zero status.
 set -e
 
-# Replace 'versus_rex' with the name of the database you want to create.
+# Database name and user credentials
 DATABASE_NAME="versus_rex"
-
-# Replace 'myusername' with the username you want to create.
 USERNAME="oedypus"
-
-# Replace 'mypassword' with the password for the new user.
 PASSWORD="Zarathustra22!"
 
 # Create a new PostgreSQL role.
 echo "Creating role..."
-sudo -u postgres psql -c "CREATE ROLE $USERNAME WITH LOGIN PASSWORD '$PASSWORD';"
+sudo -u postgres createuser --login --password "$PASSWORD" "$USERNAME"
 
 # Alter role to set superuser or necessary privileges.
-echo "Altering role..."
+echo "Altering role to superuser and allowing database creation..."
 sudo -u postgres psql -c "ALTER ROLE $USERNAME SUPERUSER CREATEDB;"
 
 # Create a new database with the new role as the owner.
 echo "Creating database..."
-sudo -u postgres psql -c "CREATE DATABASE $DATABASE_NAME WITH OWNER $USERNAME;"
+sudo -u postgres createdb --owner="$USERNAME" "$DATABASE_NAME"
 
 # Grant all privileges of the new database to the new role.
 echo "Granting privileges to user on database..."
 sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE $DATABASE_NAME TO $USERNAME;"
 
 # Optionally, set the default schema search path for the new role.
+# Replace 'my_schema' with the schema you will use.
 echo "Setting schema search path..."
-sudo -u postgres psql -c "ALTER ROLE $USERNAME SET search_path TO my_schema, public;"
-
-# Ensure the new role does not have more privileges than necessary.
-# If the role should not be a superuser, comment out or remove the ALTER ROLE line above
-# and uncomment and customize the line below according to your needs.
-# sudo -u postgres psql -c "ALTER ROLE $USERNAME NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION;"
+sudo -u postgres psql -d "$DATABASE_NAME" -c "ALTER ROLE $USERNAME SET search_path TO my_schema, public;"
+
+# Create the applied_migrations table to track migration history
+echo "Creating applied_migrations table to track migrations..."
+sudo -u postgres psql -d "$DATABASE_NAME" -c "
+BEGIN;
+CREATE TABLE IF NOT EXISTS applied_migrations (
+    migration_name TEXT PRIMARY KEY,
+    applied_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+COMMIT;
+"
 
 echo "PostgreSQL environment initialization complete."

postgresql/schemas/core_logic_template_11-05-23_03_create_tables.sql

@@ -0,0 +1,73 @@
+BEGIN;
+
+-- Creating the 'emails' table within the 'email_schema'.
+CREATE TABLE IF NOT EXISTS email_schema.emails (
+  email_id BIGSERIAL PRIMARY KEY,
+  subject TEXT,
+  sender VARCHAR(100) CHECK (sender ~* '^[^@]+@[^@]+\.[^@]+$'), -- Regex for basic email validation
+  recipient VARCHAR(100) CHECK (recipient ~* '^[^@]+@[^@]+\.[^@]+$'),
+  sent_time TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW(), -- Default to the current timestamp
+  body_text TEXT,
+  ip_address INET,
+  x_originating_ip INET,
+  received TEXT,
+  user_agent VARCHAR(255),
+  authentication_results TEXT,
+  dkim_signature TEXT,
+  sender_user_id INT,
+  recipient_user_id INT
+);
+
+-- Adding foreign key constraints to the 'emails' table.
+ALTER TABLE email_schema.emails
+  ADD CONSTRAINT fk_emails_sender_user_id
+  FOREIGN KEY (sender_user_id) REFERENCES email_schema.users(user_id)
+  ON DELETE SET NULL,
+  ADD CONSTRAINT fk_emails_recipient_user_id
+  FOREIGN KEY (recipient_user_id) REFERENCES email_schema.users(user_id)
+  ON DELETE SET NULL;
+
+-- Creating the 'attachments' table within the 'email_schema'.
+-- Note: We are ensuring that there is NO CASCADE on delete.
+CREATE TABLE IF NOT EXISTS email_schema.attachments (
+  attachment_id BIGSERIAL PRIMARY KEY,
+  email_id BIGINT NOT NULL,
+  file_name TEXT NOT NULL,
+  file_type TEXT NOT NULL, -- assuming every attachment must have a file type
+  file_size BIGINT NOT NULL CHECK (file_size >= 0), -- assuming file size must be non-negative
+  content BYTEA -- consider storing only a reference to the file location if they're large
+);
+
+-- Foreign key constraint for attachments referencing emails without cascading deletes.
+ALTER TABLE email_schema.attachments
+  ADD CONSTRAINT fk_attachments_email_id
+  FOREIGN KEY (email_id) REFERENCES email_schema.emails(email_id)
+  ON DELETE RESTRICT;
+
+-- Creating the 'email_attachment_mapping' table for a many-to-many relationship between emails and attachments.
+-- Note: We are ensuring that there is NO CASCADE on delete.
+CREATE TABLE IF NOT EXISTS email_schema.email_attachment_mapping (
+  email_id BIGINT NOT NULL,
+  attachment_id BIGINT NOT NULL,
+  PRIMARY KEY (email_id, attachment_id),
+  CONSTRAINT fk_email_attachment_mapping_email_id FOREIGN KEY (email_id) REFERENCES email_schema.emails(email_id) ON DELETE RESTRICT,
+  CONSTRAINT fk_email_attachment_mapping_attachment_id FOREIGN KEY (attachment_id) REFERENCES email_schema.attachments(attachment_id) ON DELETE RESTRICT
+);
+
+-- Creating the 'email_status' table to track the state of emails.
+-- Note: We are ensuring that there is NO CASCADE on delete.
+CREATE TABLE IF NOT EXISTS email_schema.email_status (
+  status_id SERIAL PRIMARY KEY,
+  email_id BIGINT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('sent', 'received', 'read', 'error', 'pending')), -- Add more statuses as needed
+  updated_at TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Foreign key constraint for email_status referencing emails without cascading deletes.
+ALTER TABLE email_schema.email_status
+  ADD CONSTRAINT fk_email_status_email_id
+  FOREIGN KEY (email_id) REFERENCES email_schema.emails(email_id)
+  ON DELETE RESTRICT;
+
+-- Commit the changes if all operations are successful.
+COMMIT;

postgresql/schemas/v1_11-05-23_03_create_tables.sql

@@ -0,0 +1,139 @@
+BEGIN;
+
+-- Creating the 'emails' table within the 'email_schema' to store email information.
+CREATE TABLE IF NOT EXISTS email_schema.emails (
+  email_id BIGSERIAL PRIMARY KEY,              -- A unique identifier for each email, BIGSERIAL for large number of records.
+  subject TEXT,                               -- The subject of the email.
+  sender VARCHAR(100),                        -- The email address of the sender, assuming an email address won't exceed 100 characters.
+  recipient VARCHAR(100),                     -- The email address of the recipient, same assumption as sender.
+  sent_time TIMESTAMP WITHOUT TIME ZONE,      -- The time the email was sent. Assuming timezone is handled application-side.
+  body_text TEXT,                             -- The body of the email.
+  ip_address INET,                            -- The IP address from which the email was sent, using INET type for proper IP storage.
+  x_originating_ip INET,                      -- The originating IP address if provided, also INET type.
+  received TEXT,                              -- To store the full 'Received' header chain.
+  user_agent VARCHAR(255),                    -- The user agent information if available.
+  authentication_results TEXT,                -- To store 'Authentication-Results' header information.
+  dkim_signature TEXT                         -- To store 'DKIM-Signature' header information.
+);
+
+-- Indexes on the 'emails' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_emails_sender ON email_schema.emails(sender);
+CREATE INDEX IF NOT EXISTS idx_emails_recipient ON email_schema.emails(recipient);
+-- Additional indexes omitted for brevity, but should be reviewed for actual query patterns.
+
+
+- Creating the 'attachments' table within the 'email_schema'.
+CREATE TABLE IF NOT EXISTS email_schema.attachments (
+  attachment_id BIGSERIAL PRIMARY KEY,        -- A unique identifier for each attachment, BIGSERIAL for large datasets.
+  email_id BIGINT NOT NULL REFERENCES email_schema.emails(email_id) ON DELETE CASCADE, -- Using BIGINT to match the email_id type.
+  file_name TEXT NOT NULL,                    -- The file name of the attachment.
+  file_type TEXT,                             -- The MIME type of the attachment.
+  file_size BIGINT,                           -- The size of the attachment file in bytes, BIGINT to handle large files.
+  content BYTEA                               -- To store the actual content of the attachment if needed.
+);
+
+-- Indexes on the 'attachments' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_attachments_email_id ON email_schema.attachments(email_id);
+CREATE INDEX IF NOT EXISTS idx_attachments_file_name ON email_schema.attachments(file_name);
+-- Additional indexes omitted for brevity.
+
+-- Creating the 'email_attachment_mapping' table if a many-to-many relationship is needed.
+-- Assuming from previous parts of the conversation that this might be needed.
+CREATE TABLE IF NOT EXISTS email_schema.email_attachment_mapping (
+  email_id BIGINT,
+  attachment_id BIGINT,
+  PRIMARY KEY (email_id, attachment_id),
+  FOREIGN KEY (email_id) REFERENCES email_schema.emails(email_id) ON DELETE CASCADE,
+  FOREIGN KEY (attachment_id) REFERENCES email_schema.attachments(attachment_id) ON DELETE CASCADE
+);
+
+-- Indexes for the 'email_attachment_mapping' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_email_attachment_mapping_email_id ON email_schema.email_attachment_mapping(email_id);
+CREATE INDEX IF NOT EXISTS idx_email_attachment_mapping_attachment_id ON email_schema.email_attachment_mapping(attachment_id);
+
+-- Creating the 'email_status' table to track the state of emails.
+CREATE TABLE IF NOT EXISTS email_schema.email_status (
+  status_id SERIAL PRIMARY KEY,                -- A unique identifier for each status.
+  email_id BIGINT NOT NULL REFERENCES email_schema.emails(email_id) ON DELETE CASCADE,
+  status TEXT NOT NULL,                        -- The status of the email (e.g., 'sent', 'received', 'read').
+  updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP -- Timestamp of the status update.
+);
+
+-- Index on the 'email_status' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_email_status_email_id ON email_schema.email_status(email_id);
+CREATE INDEX IF NOT EXISTS idx_email_status_status ON email_schema.email_status(status);
+
+-- Creating the 'users' table if user management is part of the scope.
+CREATE TABLE IF NOT EXISTS email_schema.users (
+  user_id SERIAL PRIMARY KEY,                  -- A unique identifier for each user.
+  email VARCHAR(255) UNIQUE NOT NULL,          -- The user's email address, unique to ensure no duplicates.
+  created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP -- Timestamp of the user creation.
+);
+
+-- Index on the 'users' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_users_email ON email_schema.users(email);
+
+-- Altering the 'emails' table to link with the 'users' table, assuming this association is required.
+ALTER TABLE email_schema.emails
+ADD COLUMN sender_user_id INT REFERENCES email_schema.users(user_id),
+ADD COLUMN recipient_user_id INT REFERENCES email_schema.users(user_id);
+
+-- Update the index after adding the new columns.
+DROP INDEX IF EXISTS email_schema.idx_emails_sender;
+DROP INDEX IF EXISTS email_schema.idx_emails_recipient;
+CREATE INDEX idx_emails_sender_user_id ON email_schema.emails(sender_user_id);
+CREATE INDEX idx_emails_recipient_user_id ON email_schema.emails(recipient_user_id);
+
+-- Assuming a logging or audit mechanism is desired.
+-- Creating a table to log actions or changes in the database.
+CREATE TABLE IF NOT EXISTS email_schema.audit_log (
+  log_id SERIAL PRIMARY KEY,                   -- A unique identifier for each log entry.
+  action TEXT NOT NULL,                        -- The action performed (e.g., 'CREATE', 'UPDATE', 'DELETE').
+  performed_by INT REFERENCES email_schema.users(user_id), -- The user who performed the action.
+  performed_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, -- Timestamp of the action.
+  detail TEXT                                  -- Details of the action performed.
+);
+
+-- Index on the 'audit_log' table to improve query performance.
+CREATE INDEX IF NOT EXISTS idx_audit_log_performed_by ON email_schema.audit_log(performed_by);
+CREATE INDEX IF NOT EXISTS idx_audit_log_performed_at ON email_schema.audit_log(performed_at);
+
+-- Adding a function to track changes for the audit log, assuming this is required.
+CREATE OR REPLACE FUNCTION email_schema.audit_log_trigger()
+RETURNS TRIGGER AS $$
+BEGIN
+  INSERT INTO email_schema.audit_log(action, performed_by, detail)
+  VALUES (TG_OP, NEW.sender_user_id, 'Email ' || TG_OP || ' with ID ' || NEW.email_id);
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+-- Creating a trigger for the 'emails' table to automatically log inserts/updates/deletes
+CREATE TRIGGER emails_audit_trigger
+AFTER INSERT OR UPDATE OR DELETE ON email_schema.emails
+FOR EACH ROW EXECUTE FUNCTION email_schema.audit_log_trigger();
+
+-- For instance, adding a routine to clean up old audit logs
+CREATE OR REPLACE FUNCTION email_schema.cleanup_audit_logs() RETURNS VOID AS $$
+BEGIN
+  DELETE FROM email_schema.audit_log WHERE performed_at < NOW() - INTERVAL '90 days';
+END;
+$$ LANGUAGE plpgsql;
+
+-- Schedule the cleanup function to run periodically, if required
+-- This requires setting up a job scheduler like pgAgent or an external cron job
+
+-- Finalize the schema with any additional grants or role assignments
+-- This is where you grant specific privileges to different database roles
+-- Example:
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA email_schema TO my_role;
+
+-- If there are any views, sequences or other schema objects, permissions for those should be set here
+
+-- Commit the transaction if everything above is successful
+COMMIT;
+
+-- The script ends here. The database is now set up with tables, indices, functions, and triggers.
+-- It's prepared for application integration and further development.
+
+