FootprintAI/Containarium: Hundred of containers living together with LXC container for saving resources and make the configuration easier

cd terraform/gce
cp examples/single-server-spot.tfvars terraform.tfvars
vim terraform.tfvars  # Add your project_id and SSH keys
terraform init
terraform apply

cp examples/horizontal-scaling-3-servers.tfvars terraform.tfvars
vim terraform.tfvars  # Configure
terraform apply

cp examples/horizontal-scaling-5-servers.tfvars terraform.tfvars
terraform apply

# Build containarium CLI for Linux
make build-linux

# Copy to jump server(s)
scp bin/containarium-linux-amd64 admin@<jump-server-ip>:/tmp/
ssh admin@<jump-server-ip>
sudo mv /tmp/containarium-linux-amd64 /usr/local/bin/containarium
sudo chmod +x /usr/local/bin/containarium

# Build containarium for your platform
make build  # macOS/Linux on your laptop

# Setup daemon on server
scp bin/containarium-linux-amd64 admin@<jump-server-ip>:/tmp/
ssh admin@<jump-server-ip>
sudo mv /tmp/containarium-linux-amd64 /usr/local/bin/containarium
sudo chmod +x /usr/local/bin/containarium

# Generate mTLS certificates
sudo containarium cert generate \
    --server-ip <jump-server-ip> \
    --output-dir /etc/containarium/certs

# Start daemon (via systemd or manually)
sudo systemctl start containarium

# Copy client certificates to your machine
exit
mkdir -p ~/.config/containarium/certs
scp admin@<jump-server-ip>:/etc/containarium/certs/ca.crt,client.crt,client.key \
    ~/.config/containarium/certs/

# SSH to jump server
ssh admin@<jump-server-ip>

# Create container for a user
sudo containarium create alice --ssh-key ~/.ssh/alice.pub

# Output:
# ✓ Creating container for user: alice
# ✓ [1/7] Creating container...
# ✓ [2/7] Starting container...
# ✓ [3/7] Creating jump server account (proxy-only)...
#   ✓ Jump server account created: alice (no shell access, proxy-only)
# ✓ [4/7] Waiting for network...
#   Container IP: 10.0.3.100
# ✓ [5/7] Installing Docker, SSH, and tools...
# ✓ [6/7] Creating user: alice...
# ✓ [7/7] Adding SSH keys (including jump server key for ProxyJump)...
# ✓ Container alice-container created successfully!
#
# Container Details:
#   Name: alice-container
#   User: alice
#   IP: 10.0.3.100
#   Disk: 50GB
#   Auto-start: enabled
#
# Jump Server Account (Secure Multi-Tenant):
#   Username: alice
#   Shell: /usr/sbin/nologin (proxy-only, no shell access)
#   SSH ProxyJump: enabled
#
# SSH Access (via ProxyJump):
#   ssh alice-dev  # (after SSH config setup)

# List containers
sudo containarium list
# +------------------+---------+----------------------+------+-----------+
# | NAME             | STATE   | IPV4                 | TYPE | SNAPSHOTS |
# +------------------+---------+----------------------+------+-----------+
# | alice-container  | RUNNING | 10.0.3.100 (eth0)    | C    | 0         |
# +------------------+---------+----------------------+------+-----------+

# No SSH required - direct gRPC call with mTLS
containarium create alice --ssh-key ~/.ssh/alice.pub \
    --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs \
    --cpu 4 --memory 8GB -v

# List containers remotely
containarium list \
    --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs

# Export SSH config remotely (run on server)
ssh admin@<jump-server-ip>
sudo containarium export alice --jump-ip 35.229.246.67 >> ~/.ssh/config

# Generate new SSH key pair
ssh-keygen -t ed25519 -C "alice@company.com" -f ~/.ssh/containarium_alice

# Output:
# ~/.ssh/containarium_alice      (private key - keep secret!)
# ~/.ssh/containarium_alice.pub  (public key - share with admin)

# User sends their public key to admin
# Admin receives: alice_id_ed25519.pub

# SSH to jump server
ssh admin@<jump-server-ip>

# Create container with user's public key
sudo containarium create alice --ssh-key /path/to/alice_id_ed25519.pub

# Or if key is on admin's local machine, copy it first:
scp alice_id_ed25519.pub admin@<jump-server-ip>:/tmp/
ssh admin@<jump-server-ip>
sudo containarium create alice --ssh-key /tmp/alice_id_ed25519.pub

# This will FAIL (proxy-only account - no shell)
ssh containarium-jump
# Output: "This account is currently not available."

# This WORKS (ProxyJump to container)
ssh alice-dev
# Output: alice@alice-container:~$

ssh my-dev
# Alice is now in her Ubuntu container with Docker!

# First connection will ask to verify host key:
# The authenticity of host '10.0.3.100 (<no hostip for proxy command>)' can't be established.
# ED25519 key fingerprint is SHA256:...
# Are you sure you want to continue connecting (yes/no)? yes

# Method 1: Using incus exec
sudo incus exec alice-container -- bash -c "echo 'ssh-ed25519 AAAA...' >> /home/alice/.ssh/authorized_keys"

# Method 2: Using incus file push
echo 'ssh-ed25519 AAAA...' > /tmp/new_key.pub
sudo incus file push /tmp/new_key.pub alice-container/home/alice/.ssh/authorized_keys --mode 0600 --uid 1000 --gid 1000

# Method 3: SSH into container and add manually
ssh alice@10.0.3.100   # (from jump server)
echo 'ssh-ed25519 AAAA...' >> ~/.ssh/authorized_keys

# Overwrite authorized_keys with new key
echo 'ssh-ed25519 NEW_KEY_AAAA...' | sudo incus exec alice-container -- \
  tee /home/alice/.ssh/authorized_keys > /dev/null

# Set correct permissions
sudo incus exec alice-container -- chown alice:alice /home/alice/.ssh/authorized_keys
sudo incus exec alice-container -- chmod 600 /home/alice/.ssh/authorized_keys

# Edit authorized_keys file
sudo incus exec alice-container -- bash -c \
  "sed -i '/alice@old-laptop/d' /home/alice/.ssh/authorized_keys"

# List all authorized keys for a user
sudo incus exec alice-container -- cat /home/alice/.ssh/authorized_keys

# Show all commands
make help

# Build for current platform
make build

# Build for Linux (deployment)
make build-linux

# Generate protobuf code
make proto

# Run tests
make test

# Run E2E tests (requires GCP credentials)
export GCP_PROJECT=your-project-id
make test-e2e

# Lint and format
make lint fmt

# Build and run locally
make run-local

# Test commands
./bin/containarium create alice
./bin/containarium list
./bin/containarium info alice

# Set GCP project
export GCP_PROJECT=your-gcp-project-id

# Run full E2E test (25-30 min)
make test-e2e

# Test workflow:
# 1. Deploy infrastructure with Terraform
# 2. Wait for instance ready
# 3. Verify ZFS setup
# 4. Create container with test data
# 5. Reboot instance (stop/start)
# 6. Verify data persisted
# 7. Cleanup infrastructure

# Alice connects to her container
Jan 10 14:23:15 jump-server sshd[12345]: Accepted publickey for alice from 203.0.113.10
Jan 10 14:23:15 jump-server sshd[12345]: pam_unix(sshd:session): session opened for user alice

# Bob connects to his container
Jan 10 14:25:32 jump-server sshd[12346]: Accepted publickey for bob from 203.0.113.11
Jan 10 14:25:32 jump-server sshd[12346]: pam_unix(sshd:session): session opened for user bob

# Failed login attempt
Jan 10 14:30:01 jump-server sshd[12347]: Failed publickey for charlie from 203.0.113.12
Jan 10 14:30:05 jump-server sshd[12348]: Failed publickey for charlie from 203.0.113.12
Jan 10 14:30:09 jump-server sshd[12349]: Failed publickey for charlie from 203.0.113.12

# SSH to jump server as admin
ssh admin@<jump-server-ip>

# View all SSH connections
sudo journalctl -u sshd -f

# View connections for specific user
sudo journalctl -u sshd | grep "for alice"

# View failed login attempts
sudo journalctl -u sshd | grep "Failed"

# View connections from specific IP
sudo journalctl -u sshd | grep "from 203.0.113.10"

# Export logs for security audit
sudo journalctl -u sshd --since "2025-01-01" --until "2025-01-31" > ssh-audit-jan-2025.log

# Automatically installed by Terraform startup script
sudo apt install -y fail2ban

# Check fail2ban status
sudo fail2ban-client status

# Check SSH jail status
sudo fail2ban-client status sshd

# Output:
# Status for the jail: sshd
# |- Filter
# |  |- Currently failed:  2
# |  |- Total failed:      15
# |  `- File list:         /var/log/auth.log
# `- Actions
#    |- Currently banned:  1
#    |- Total banned:      3
#    `- Banned IP list:    203.0.113.12

# View banned IPs
sudo fail2ban-client get sshd banip

# Unban IP manually (if needed)
sudo fail2ban-client set sshd unbanip 203.0.113.12

# View fail2ban activity
sudo tail -f /var/log/fail2ban.log

# Example output:
# 2025-01-10 14:30:15,123 fail2ban.filter  [12345]: INFO    [sshd] Found 203.0.113.12 - 2025-01-10 14:30:09
# 2025-01-10 14:30:20,456 fail2ban.actions [12346]: NOTICE  [sshd] Ban 203.0.113.12
# 2025-01-10 15:30:20,789 fail2ban.actions [12347]: NOTICE  [sshd] Unban 203.0.113.12

#!/bin/bash

echo "=== Containarium Security Status ==="
echo ""

echo "📊 Active SSH Sessions:"
who
echo ""

echo "🚫 Banned IPs (fail2ban):"
sudo fail2ban-client status sshd | grep "Banned IP"
echo ""

echo "⚠️  Recent Failed Login Attempts:"
sudo journalctl -u sshd --since "1 hour ago" | grep "Failed" | tail -10
echo ""

echo "✅ Successful Logins (last hour):"
sudo journalctl -u sshd --since "1 hour ago" | grep "Accepted publickey" | tail -10
echo ""

echo "👥 Unique Users Connected Today:"
sudo journalctl -u sshd --since "today" | grep "Accepted publickey" | \
  awk 'print $9' | sort -u

# Make executable
sudo chmod +x /usr/local/bin/security-monitor.sh

# Run manually
sudo /usr/local/bin/security-monitor.sh

# Add to cron for daily reports
echo "0 9 * * * /usr/local/bin/security-monitor.sh | mail -s 'Daily Security Report' admin@company.com" | sudo crontab -

# Count connections per user
sudo journalctl -u sshd --since "today" | grep "Accepted publickey" | \
  awk 'print $9' | sort | uniq -c | sort -rn

# Output:
#  45 alice
#  32 bob
#  18 charlie
#   5 david

# View all of Alice's connections
sudo journalctl -u sshd | grep "for alice" | grep "Accepted publickey"

# Find when Bob last connected
sudo journalctl -u sshd | grep "for bob" | grep "Accepted publickey" | tail -1

# fail2ban detects excessive failed attempts from alice's IP
2025-01-10 15:00:00 fail2ban.filter [12345]: INFO [sshd] Found alice from 203.0.113.10
2025-01-10 15:00:05 fail2ban.filter [12346]: INFO [sshd] Found alice from 203.0.113.10
2025-01-10 15:00:10 fail2ban.filter [12347]: INFO [sshd] Found alice from 203.0.113.10
2025-01-10 15:00:15 fail2ban.actions [12348]: NOTICE [sshd] Ban 203.0.113.10

# Result:
# ✅ Alice's IP is banned (her laptop is blocked)
# ✅ Bob, Charlie, and other users are NOT affected
# ✅ Service continues for everyone else
# ✅ Admin can investigate Alice's account specifically

# ❌ Can't tell which user is causing the issue
# ❌ Banning the IP might affect legitimate users behind NAT
# ❌ No per-user accountability

# Export all SSH activity for user 'alice' in January
sudo journalctl -u sshd --since "2025-01-01" --until "2025-02-01" | \
  grep "for alice" > alice-ssh-audit-jan-2025.log

# Export all failed login attempts
sudo journalctl -u sshd --since "2025-01-01" --until "2025-02-01" | \
  grep "Failed" > failed-logins-jan-2025.log

# Export fail2ban bans
sudo fail2ban-client get sshd banhistory > ban-history-jan-2025.log

# Generate SSH key pair
ssh-keygen -t ed25519 -C "alice@company.com" -f ~/.ssh/containarium_alice

# Output:
# Generating public/private ed25519 key pair.
# Enter passphrase (empty for no passphrase): [optional]
# Your identification has been saved in ~/.ssh/containarium_alice
# Your public key has been saved in ~/.ssh/containarium_alice.pub

# View and copy public key to send to admin
cat ~/.ssh/containarium_alice.pub
# ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJqL+XYZ... alice@company.com

# Save user's public key to file
echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJqL... alice@company.com' > /tmp/alice.pub

# SSH to jump server
ssh admin@<jump-server-ip>

# Create container with user's public key
# This automatically:
#   1. Creates jump server account for alice (proxy-only, no shell)
#   2. Creates alice-container with SSH access
#   3. Sets up SSH keys for both
sudo containarium create alice --ssh-key /tmp/alice.pub

# Output:
# ✓ Creating jump server account: alice (proxy-only)
# ✓ Creating container for user: alice
# ✓ Container started: alice-container
# ✓ IP Address: 10.0.3.100
# ✓ Installing Docker and dev tools
# ✓ Container alice-container created successfully!
#
# ✓ Jump server account: alice@35.229.246.67 (proxy-only, no shell)
# ✓ Container access: alice@10.0.3.100
#
# Send this to user:
#   Jump Server: 35.229.246.67 (user: alice)
#   Container IP: 10.0.3.100
#   Username: alice

# Enable auto-start for spot instance recovery
sudo incus config set alice-container boot.autostart true

# Admin exports SSH configuration
sudo containarium export alice --jump-ip 35.229.246.67 --key ~/.ssh/containarium_alice > alice-ssh-config.txt

# Send alice-ssh-config.txt to user via email/Slack

# Add exported config to your SSH config
cat alice-ssh-config.txt >> ~/.ssh/config

# Connect to container
ssh alice-dev

# You're now in your container!
alice@alice-container:~$ docker run hello-world
alice@alice-container:~$ sudo apt install vim git tmux

# Add to ~/.ssh/config
vim ~/.ssh/config

# Paste the configuration provided by admin

# Connect to container
ssh alice-dev

# First time: verify host key
# The authenticity of host '10.0.3.100' can't be established.
# ED25519 key fingerprint is SHA256:...
# Are you sure you want to continue connecting (yes/no)? yes

# You're now in your container!
alice@alice-container:~$ docker run hello-world
alice@alice-container:~$ sudo apt install vim git tmux

# On second laptop, generate new key
ssh-keygen -t ed25519 -C "alice@home-laptop" -f ~/.ssh/containarium_alice_home

# Send new public key to admin
cat ~/.ssh/containarium_alice_home.pub

# Add second key to container (keeps existing keys)
NEW_KEY='ssh-ed25519 AAAAC3... alice@home-laptop'
sudo incus exec alice-container -- bash -c \
  "echo '$NEW_KEY' >> /home/alice/.ssh/authorized_keys"

# Execute directly on the jump server (requires sudo)
sudo containarium create alice --ssh-key ~/.ssh/alice.pub
sudo containarium list
sudo containarium delete bob

# Execute from anywhere (laptop, CI/CD, etc.)
containarium create alice --ssh-key ~/.ssh/alice.pub \
    --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs

containarium list --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs

# Run as systemd service on the jump server
containarium daemon --address 0.0.0.0 --port 50051 --mtls

# Systemd service configuration
sudo systemctl start containarium
sudo systemctl enable containarium
sudo systemctl status containarium

# On server: Generate server and client certificates
containarium cert generate \
    --server-ip 35.229.246.67 \
    --output-dir /etc/containarium/certs

# Copy client certificates to local machine
scp admin@35.229.246.67:/etc/containarium/certs/ca.crt,client.crt,client.key \
    ~/.config/containarium/certs/

# Test remote connection
containarium list \
    --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs

# Basic usage
sudo containarium create <username> --ssh-key <path-to-public-key>

# Example
sudo containarium create alice --ssh-key ~/.ssh/alice.pub

# With custom disk quota
sudo containarium create bob --ssh-key ~/.ssh/bob.pub --disk-quota 50GB

# Enable auto-start on boot
sudo containarium create charlie --ssh-key ~/.ssh/charlie.pub --autostart

# List all containers
sudo containarium list

# Example output
NAME              STATUS    IP            QUOTA   AUTOSTART
alice-container   Running   10.0.3.100    20GB    Yes
bob-container     Running   10.0.3.101    50GB    Yes
charlie-container Stopped   -             20GB    No

# Get detailed information
sudo containarium info alice

# Example output
Container: alice-container
Status: Running
User: alice
IP Address: 10.0.3.100
Disk Quota: 20GB
Disk Used: 4.2GB (21%)
Memory: 512MB / 2GB
CPU Usage: 5%
Uptime: 3 days
Auto-start: Enabled

# Delete container (with confirmation)
sudo containarium delete alice

# Force delete (no confirmation)
sudo containarium delete bob --force

# Delete with data backup
sudo containarium delete charlie --backup

# Resize CPU only
sudo containarium resize alice --cpu 4

# Resize memory only
sudo containarium resize alice --memory 8GB

# Resize disk only
sudo containarium resize alice --disk 100GB

# Resize all three at once
sudo containarium resize alice --cpu 4 --memory 8GB --disk 100GB

# With verbose output
sudo containarium resize alice --cpu 8 --memory 16GB -v

# Set specific number of cores
sudo containarium resize alice --cpu 4

# Set CPU range (flexible allocation)
sudo containarium resize alice --cpu 2-4

# Pin to specific CPU cores (performance)
sudo containarium resize alice --cpu 0-3

# Gigabytes
sudo containarium resize alice --memory 8GB

# Megabytes
sudo containarium resize alice --memory 4096MB

# Gibibytes (binary)
sudo containarium resize alice --memory 8GiB

$ sudo containarium resize alice --cpu 4 --memory 8GB -v
Resizing container: alice-container
  Setting CPU limit: 4
  Setting memory limit: 8GB
  ✓ Resources updated successfully (no restart required)

✓ Container alice-container resized successfully!

Updated configuration:
  CPU:    4
  Memory: 8GB

# Export to stdout (copy/paste to ~/.ssh/config)
sudo containarium export alice --jump-ip 35.229.246.67

# Export to file
sudo containarium export alice --jump-ip 35.229.246.67 --output ~/.ssh/config.d/containarium-alice

# With custom SSH key path
sudo containarium export alice --jump-ip 35.229.246.67 --key ~/.ssh/containarium_alice

# Append directly to SSH config
sudo containarium export alice --jump-ip 35.229.246.67 >> ~/.ssh/config

# After exporting, connect with:
ssh alice-dev

# User generates their own key pair
ssh-keygen -t ed25519 -C "user@company.com" -f ~/.ssh/containarium

# Output files:
# ~/.ssh/containarium      (private - never share!)
# ~/.ssh/containarium.pub  (public - give to admin)

# View public key (to send to admin)
cat ~/.ssh/containarium.pub

# Method 1: Admin has key file locally
sudo containarium create alice --ssh-key /path/to/alice.pub

# Method 2: Admin receives key via secure channel
# User sends their public key:
cat ~/.ssh/containarium.pub
# ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJqL... alice@company.com

# Admin creates container with key inline
echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJqL... alice@company.com' > /tmp/alice.pub
sudo containarium create alice --ssh-key /tmp/alice.pub

# Method 3: Multiple users with different keys
sudo containarium create alice --ssh-key /tmp/alice.pub
sudo containarium create bob --ssh-key /tmp/bob.pub
sudo containarium create charlie --ssh-key /tmp/charlie.pub

# Add additional key (keep existing keys)
NEW_KEY='ssh-ed25519 AAAAC3... user@laptop'
sudo incus exec alice-container -- bash -c \
  "echo '$NEW_KEY' >> /home/alice/.ssh/authorized_keys"

# Verify key was added
sudo incus exec alice-container -- cat /home/alice/.ssh/authorized_keys

# Replace all keys with new key
NEW_KEY='ssh-ed25519 AAAAC3... alice@new-laptop'
echo "$NEW_KEY" | sudo incus exec alice-container -- \
  tee /home/alice/.ssh/authorized_keys > /dev/null

# Fix permissions
sudo incus exec alice-container -- chown alice:alice /home/alice/.ssh/authorized_keys
sudo incus exec alice-container -- chmod 600 /home/alice/.ssh/authorized_keys

# Add work laptop key
WORK_KEY='ssh-ed25519 AAAAC3... alice@work-laptop'
sudo incus exec alice-container -- bash -c \
  "echo '$WORK_KEY' >> /home/alice/.ssh/authorized_keys"

# Add home laptop key
HOME_KEY='ssh-ed25519 AAAAC3... alice@home-laptop'
sudo incus exec alice-container -- bash -c \
  "echo '$HOME_KEY' >> /home/alice/.ssh/authorized_keys"

# View all keys
sudo incus exec alice-container -- cat /home/alice/.ssh/authorized_keys
# ssh-ed25519 AAAAC3... alice@work-laptop
# ssh-ed25519 AAAAC3... alice@home-laptop

# Remove key by comment (last part of key)
sudo incus exec alice-container -- bash -c \
  "sed -i '/alice@old-laptop/d' /home/alice/.ssh/authorized_keys"

# Remove key by fingerprint pattern
sudo incus exec alice-container -- bash -c \
  "sed -i '/AAAAC3NzaC1lZDI1NTE5AAAAIAbc123/d' /home/alice/.ssh/authorized_keys"

# Check authorized_keys permissions
sudo incus exec alice-container -- ls -la /home/alice/.ssh/
# Should show:
# drwx------ 2 alice alice 4096 ... .ssh
# -rw------- 1 alice alice  123 ... authorized_keys

# Fix permissions if wrong
sudo incus exec alice-container -- chown -R alice:alice /home/alice/.ssh
sudo incus exec alice-container -- chmod 700 /home/alice/.ssh
sudo incus exec alice-container -- chmod 600 /home/alice/.ssh/authorized_keys

# Test SSH from jump server
ssh -v alice@10.0.3.100
# -v shows verbose output for debugging

# Check SSH logs in container
sudo incus exec alice-container -- tail -f /var/log/auth.log

# Execute command in container
sudo incus exec alice-container -- df -h

# Shell into container
sudo incus exec alice-container -- su - alice

# View container logs
sudo incus console alice-container --show-log

# Snapshot container
sudo incus snapshot alice-container snap1

# Restore snapshot
sudo incus restore alice-container snap1

# Copy container
sudo incus copy alice-container alice-backup

# Set memory limit
sudo incus config set alice-container limits.memory 4GB

# Set CPU limit
sudo incus config set alice-container limits.cpu 2

# View container metrics
sudo incus info alice-container

# Resize disk quota
sudo containarium resize alice --disk-quota 100GB

cd terraform/gce

# Initialize Terraform
terraform init

# Preview changes
terraform plan

# Deploy infrastructure
terraform apply

# Deploy with custom variables
terraform apply -var-file=examples/horizontal-scaling-3-servers.tfvars

# Show outputs
terraform output

# Get specific output
terraform output jump_server_ip

# Update infrastructure
terraform apply

# Destroy specific resource
terraform destroy -target=google_compute_instance.jump_server_spot[0]

# Destroy everything
terraform destroy

# Import existing resource
terraform import google_compute_instance.jump_server projects/my-project/zones/us-central1-a/instances/my-instance

# Refresh state
terraform refresh

# Backup ZFS pool
sudo zfs snapshot incus-pool@backup-$(date +%Y%m%d)

# List snapshots
sudo zfs list -t snapshot

# Rollback to snapshot
sudo zfs rollback incus-pool@backup-20240115

# Export container
sudo incus export alice-container alice-backup.tar.gz

# Import container
sudo incus import alice-backup.tar.gz

# Check ZFS pool status
sudo zpool status

# Check disk usage
sudo zfs list

# Check container resource usage
sudo incus list --columns ns4mDcup

# View system load
htop

# Check Incus daemon status
sudo systemctl status incus

# Check what google_guest_agent is doing
sudo journalctl -u google-guest-agent --since "5 minutes ago" | grep -E "account|user|Updating"

# Temporarily disable account management (if needed)
sudo systemctl stop google-guest-agent
sudo containarium create alice --ssh-key ~/.ssh/alice.pub
sudo systemctl start google-guest-agent

# Or wait and retry - agent usually releases lock within 30-60 seconds

# View Incus logs
sudo journalctl -u incus -f

# Check container network
sudo incus network list
sudo incus network show incusbr0

# Restart Incus daemon
sudo systemctl restart incus

# Check startup script logs (GCE)
gcloud compute instances get-serial-port-output <instance-name> --zone=<zone>

# Verify ZFS health
sudo zpool scrub incus-pool
sudo zpool status -v

# View daemon status
sudo systemctl status containarium

# View daemon logs
sudo journalctl -u containarium -f

# Restart daemon
sudo systemctl restart containarium

# Test daemon connection (with mTLS)
containarium list \
    --server 35.229.246.67:50051 \
    --certs-dir ~/.config/containarium/certs

# Install grpcurl if needed
go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest

# List services (with mTLS)
grpcurl -cacert /etc/containarium/certs/ca.crt \
    -cert /etc/containarium/certs/client.crt \
    -key /etc/containarium/certs/client.key \
    35.229.246.67:50051 list

# Create container via gRPC (with mTLS)
grpcurl -cacert /etc/containarium/certs/ca.crt \
    -cert /etc/containarium/certs/client.crt \
    -key /etc/containarium/certs/client.key \
    -d '"username": "alice", "ssh_keys": ["ssh-ed25519 AAA..."]' \
    35.229.246.67:50051 containarium.v1.ContainerService/CreateContainer

# Create multiple containers
for user in alice bob charlie; do
  sudo containarium create $user --ssh-key ~/.ssh/$user.pub
done

# Enable autostart for all containers
sudo incus list --format csv -c n | while read name; do
  sudo incus config set $name boot.autostart true
done

# Snapshot all containers
for container in $(sudo incus list --format csv -c n); do
  sudo incus snapshot $container "backup-$(date +%Y%m%d)"
done

cd terraform/gce
terraform init
terraform plan
terraform apply

# Spot VM terminated → Containers stop
# VM restarts → Containers auto-start (boot.autostart=true)
# Downtime: 2-5 minutes
# Data: Preserved on persistent disk

# Automatic snapshots enabled by default
enable_disk_snapshots = true

# 30-day retention
# Point-in-time recovery available

# SSH traffic distributed across healthy servers
# Session affinity keeps users on same server
# Health checks on port 22

# User generates new key
ssh-keygen -t ed25519 -C "alice@company.com" -f ~/.ssh/containarium_alice_new

# Admin replaces old key with new key
NEW_KEY='ssh-ed25519 AAAAC3... alice@new-laptop'
echo "$NEW_KEY" | sudo incus exec alice-container -- \
  tee /home/alice/.ssh/authorized_keys > /dev/null

# Admin adds second key for same user
sudo incus exec alice-container -- bash -c \
  "echo 'ssh-ed25519 AAAAC3... alice@home-laptop' >> /home/alice/.ssh/authorized_keys"

# Inside your container
docker run hello-world
docker-compose up -d

# Increase quota to 50GB
sudo containarium resize alice --disk-quota 50GB

# SSH to your container
ssh my-dev

# Install packages as usual
sudo apt update
sudo apt install vim git tmux htop

# Or use Docker
docker run -it ubuntu bash

# Clone repo
git clone https://github.com/footprintai/Containarium.git
cd Containarium/terraform/gce

# Choose your size and configure
cp examples/single-server-spot.tfvars terraform.tfvars
vim terraform.tfvars  # Add: project_id, admin_ssh_keys, allowed_ssh_sources

# Deploy to GCP
terraform init
terraform apply  # Creates VM with Incus pre-installed

# Save the jump server IP from output!

# Build for Linux
cd ../..
make build-linux

# Copy to jump server
scp bin/containarium-linux-amd64 admin@<jump-server-ip>:/tmp/

# SSH and install
ssh admin@<jump-server-ip>
sudo mv /tmp/containarium-linux-amd64 /usr/local/bin/containarium
sudo chmod +x /usr/local/bin/containarium
exit

# User generates their key (on their local machine)
ssh-keygen -t ed25519 -C "alice@company.com" -f ~/.ssh/containarium_alice
# User sends public key file to admin: ~/.ssh/containarium_alice.pub

# SSH to jump server
ssh admin@<jump-server-ip>

# Save users' public keys (received from users)
echo 'ssh-ed25519 AAAAC3... alice@company.com' > /tmp/alice.pub
echo 'ssh-ed25519 AAAAC3... bob@company.com' > /tmp/bob.pub

# Create containers with users' keys
sudo containarium create alice --ssh-key /tmp/alice.pub --image images:ubuntu/24.04
sudo containarium create bob --ssh-key /tmp/bob.pub --image images:ubuntu/24.04
sudo containarium create charlie --ssh-key /tmp/charlie.pub --image images:ubuntu/24.04

# Output:
# ✓ Container alice-container created successfully!
# ✓ Jump server account: alice (proxy-only, no shell access)
# IP Address: 10.0.3.166

# Enable auto-start (survive spot instance restarts)
sudo incus config set alice-container boot.autostart true
sudo incus config set bob-container boot.autostart true
sudo incus config set charlie-container boot.autostart true

# Export SSH configs for users
sudo containarium export alice --jump-ip <jump-server-ip> > alice-ssh-config.txt
sudo containarium export bob --jump-ip <jump-server-ip> > bob-ssh-config.txt
sudo containarium export charlie --jump-ip <jump-server-ip> > charlie-ssh-config.txt

# Send config files to users
# List all containers
sudo containarium list

# Alice on her laptop
cat alice-ssh-config.txt >> ~/.ssh/config

# Connect!
ssh alice-dev
# Alice is now in her Ubuntu container with Docker!
docker run hello-world