Disk configuration for 'longing'

home | airgap | charisma | verve | trail | commitment | discernment | lore


For more info see 'longing'.

nvme0n1 (1.8TB)
S6Z2NJ0TA15369X
nvme0n1p1 (1MB)
grub
nvme0n1p2 (2GB)
md0
nvme0n1p3 (48GB)
data:cache
nvme0n1p4 (128GB)
md1
nvme0n1p5 (1.6TB)
fast
nvme1n1 (1.8TB)
S6Z2NJ0TA15361M
nvme1n1p1 (1MB)
grub
nvme1n1p2 (2GB)
md0
nvme1n1p3 (48GB)
data:cache
nvme1n1p4 (128GB)
md1
nvme1n1p5 (1.6TB)
fast
sda (14.6TB)
2MKTESDB
data
sdb (14.6TB)
2BKRRZET
data
sdc (14.6TB)
2BKRHVRT
data
Device Drive Model Commissioned Sequential Random WRL Temp °C Data Sheet
nvme[01]n1 Samsung 990 PRO PCIe 4.0 NVMe M.2 SSD 2TB MZ-V9P2T0BW 2022-12-10 3,379 MB/s 138 MB/s 1,200 TBW 0°C to 70°C online
sd[abc] WD Ultrastar DC HC550 16TB WDC WUH721816AL 2022-12-10 173 MB/s 8.9 MB/s 550 TBW 5°C to 60°C download

fstab

Device RAID Capacity File-system Mount
md0 RAID1 2GB ext4 /boot
md1 RAID1 128GB btrfs /

ZFS zpools

Pool RAID Capacity Compression Dedup
fast RAID1 1.64TB lz4 on
data RAID1 14.5TB zstd on

ZFS datasets

These are the datasets we create on our zpools.

Dataset Mount Note
fast /fast
fast/home /home
fast/home/jj5 /home/jj5
fast/mysql /var/lib/mysql no compression, no dedup
fast/scratch /fast/scratch no compression, no dedup
fast/vbox /fast/vbox
fast/session /var/log/session
data /data
data/archive /data/archive
data/backup /data/backup
data/blob /data/blob
data/dataset /data/dataset
data/host /data/host
data/image /data/image no compression, no dedup
data/image/archive /data/image/archive no compression, no dedup
data/image/backup /data/image/backup no compression, no dedup
data/image/vm /data/image/vm no compression, no dedup
data/share /data/share
data/temp /temp
data/temp/extract /temp/extract
data/temp/public /temp/public
data/temp/rubbish /temp/rubbish
data/temp/upload /temp/upload
data/temp/xfer /temp/xfer

Commands

Initial setup on 2022-12-10

#!/bin/bash

set -euo pipefail;

FAST_DISK_1=/dev/disk/by-id/nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15369X-part5
FAST_DISK_2=/dev/disk/by-id/nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15361M-part5

DATA_DISK_1=/dev/disk/by-id/scsi-SATA_WDC_WUH721816AL_2MKTESDB
DATA_DISK_2=/dev/disk/by-id/scsi-SATA_WDC_WUH721816AL_2BKRRZET
DATA_DISK_3=/dev/disk/by-id/scsi-SATA_WDC_WUH721816AL_2BKRHVRT

DATA_CACHE_1=/dev/disk/by-id/nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15361M-part3
DATA_CACHE_2=/dev/disk/by-id/nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15369X-part3

apt install zfsutils-linux

zpool create \
    -o ashift=14 -o autotrim=on \
    -O acltype=posixacl -O compression=lz4 \
    -O dnodesize=auto -O normalization=formD -O atime=off -O dedup=on \
    -O xattr=sa \
    fast mirror ${FAST_DISK_1} ${FAST_DISK_2}

zpool create \
    -O acltype=posixacl -O compression=zstd \
    -O dnodesize=auto -O normalization=formD -O atime=off -O dedup=on \
    -O xattr=sa \
    data mirror ${DATA_DISK_1} ${DATA_DISK_2} ${DATA_DISK_3}

zpool add data cache ${DATA_CACHE_1}
zpool add data cache ${DATA_CACHE_2}

zfs create fast/scratch
zfs set dedup=off fast/scratch
zfs set compression=off fast/scratch
zfs create fast/home
zfs create fast/home/jj5
zfs create fast/mysql
zfs set dedup=off fast/mysql
zfs set compression=off fast/mysql
zfs set mountpoint=/var/lib/mysql fast/mysql
zfs create fast/vbox

zfs create data/host
zfs create data/image
zfs set dedup=off data/image
zfs set compression=off data/image
zfs create data/image/archive
zfs create data/image/backup
zfs create data/image/vm
zfs create data/temp
zfs set mountpoint=/temp data/temp
zfs create data/temp/extract
zfs create data/temp/public
zfs create data/temp/rubbish
zfs create data/temp/upload
zfs create data/temp/xfer
zfs create data/archive
zfs create data/backup
zfs create data/blob
zfs create data/dataset
zfs create data/share

mv /home /home.bak

zfs set mountpoint=/home fast/home

chown jj5:jj5 /home/jj5
mv /home.bak/jj5/.[!.]* /home/jj5/
#mv /home.bak/jj5/* /home/jj5/

Photos

First install

These are some photos of the hardware and the install screen.

Second install (2022-12-23)

These are some photos taken during the reinstall.

Configuration status

-------------------
Sun Dec 11 15:26:15 [bash:5.1.16 jobs:0 error:0 time:1]
root@longing:/home/jj5
# cat /proc/mdstat
Personalities : [raid1] [linear] [multipath] [raid0] [raid6] [raid5] [raid4] [raid10]
md1 : active raid1 nvme1n1p4[0] nvme0n1p4[1]
      134085632 blocks super 1.2 [2/2] [UU]
      bitmap: 1/1 pages [4KB], 65536KB chunk

md0 : active raid1 nvme1n1p2[0] nvme0n1p2[1]
      2094080 blocks super 1.2 [2/2] [UU]

unused devices: 
-------------------
Sun Dec 11 15:26:23 [bash:5.1.16 jobs:0 error:0 time:9]
root@longing:/home/jj5
# cat /etc/fstab
# /etc/fstab: static file system information.
#
# Use 'blkid' to print the universally unique identifier for a
# device; this may be used with UUID= as a more robust way to name devices
# that works even if disks are added and removed. See fstab(5).
#
#                

# 2022-12-10 jj5 - OLD: this is used for ZFS data:cache now...
#/dev/disk/by-uuid/66fd5eeb-ff29-4ee0-a09c-cd7deead1aeb none swap sw 0 0

# / was on /dev/md1p1 during curtin installation
/dev/disk/by-id/md-uuid-2894a69f:005489a3:1f084bfc:3a6bec9a-part1 / btrfs noatime 0 1

# /boot was on /dev/md0p1 during curtin installation
/dev/disk/by-id/md-uuid-cd492c21:e9d2d334:92efcc55:42936ee9-part1 /boot ext4 noatime 0 1

# 2022-12-10 jj5 - this is our swap file, it was created by the installer...
/swap.img  none  swap  sw  0  2

-------------------
Sun Dec 11 15:26:27 [bash:5.1.16 jobs:0 error:0 time:13]
root@longing:/home/jj5
# zpool list
NAME   SIZE  ALLOC   FREE  CKPOINT  EXPANDSZ   FRAG    CAP  DEDUP    HEALTH  ALTROOT
data  14.5T   436G  14.1T        -         -     1%     2%  1.71x    ONLINE  -
fast  1.64T   119M  1.64T        -         -     0%     0%  1.79x    ONLINE  -
-------------------
Sun Dec 11 15:26:30 [bash:5.1.16 jobs:0 error:0 time:16]
root@longing:/home/jj5
# zpool status
  pool: data
 state: ONLINE
  scan: scrub repaired 0B in 00:52:45 with 0 errors on Sun Dec 11 01:16:46 2022
config:

        NAME                                                  STATE     READ WRITE CKSUM
        data                                                  ONLINE       0     0     0
          mirror-0                                            ONLINE       0     0     0
            scsi-SATA_WDC_WUH721816AL_2MKTESDB                ONLINE       0     0     0
            scsi-SATA_WDC_WUH721816AL_2BKRRZET                ONLINE       0     0     0
            scsi-SATA_WDC_WUH721816AL_2BKRHVRT                ONLINE       0     0     0
        cache
          nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15361M-part3  ONLINE       0     0     0
          nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15369X-part3  ONLINE       0     0     0

errors: No known data errors

  pool: fast
 state: ONLINE
  scan: scrub repaired 0B in 00:00:00 with 0 errors on Sun Dec 11 00:24:04 2022
config:

        NAME                                                    STATE     READ WRITE CKSUM
        fast                                                    ONLINE       0     0     0
          mirror-0                                              ONLINE       0     0     0
            nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15369X-part5  ONLINE       0     0     0
            nvme-Samsung_SSD_990_PRO_2TB_S6Z2NJ0TA15361M-part5  ONLINE       0     0     0

errors: No known data errors
-------------------
Sun Dec 11 15:26:34 [bash:5.1.16 jobs:0 error:0 time:20]
root@longing:/home/jj5
# zfs list
NAME                    USED  AVAIL     REFER  MOUNTPOINT
data                    727G  14.0T      152K  /data
data/archive             96K  14.0T       96K  /data/archive
data/backup             824K  14.0T      824K  /data/backup
data/blob                96K  14.0T       96K  /data/blob
data/dataset             96K  14.0T       96K  /data/dataset
data/host               720G  14.0T      120K  /data/host
data/image              392K  14.0T      104K  /data/image
data/image/archive       96K  14.0T       96K  /data/image/archive
data/image/backup        96K  14.0T       96K  /data/image/backup
data/image/vm            96K  14.0T       96K  /data/image/vm
data/share               96K  14.0T       96K  /data/share
data/temp               608K  14.0T      128K  /temp
data/temp/extract        96K  14.0T       96K  /temp/extract
data/temp/public         96K  14.0T       96K  /temp/public
data/temp/rubbish        96K  14.0T       96K  /temp/rubbish
data/temp/upload         96K  14.0T       96K  /temp/upload
data/temp/xfer           96K  14.0T       96K  /temp/xfer
fast                    161M  1.59T      416K  /fast
fast/home               128M  1.59T      896K  /home
fast/home/jj5           127M  1.59T      127M  /home/jj5
fast/mysql             10.8M  1.59T     10.8M  /var/lib/mysql
fast/scratch            384K  1.59T      384K  /fast/scratch
fast/vbox               384K  1.59T      384K  /fast/vbox
-------------------
Sun Dec 11 15:26:37 [bash:5.1.16 jobs:0 error:0 time:23]
root@longing:/home/jj5
# zfs get dedup
NAME                                     PROPERTY  VALUE          SOURCE
data                                     dedup     on             local
data/archive                             dedup     on             inherited from data
data/backup                              dedup     on             inherited from data
data/blob                                dedup     on             inherited from data
data/dataset                             dedup     on             inherited from data
data/host                                dedup     on             inherited from data
data/image                               dedup     off            local
data/image/archive                       dedup     off            inherited from data/image
data/image/backup                        dedup     off            inherited from data/image
data/image/vm                            dedup     off            inherited from data/image
data/share                               dedup     on             inherited from data
data/temp                                dedup     on             inherited from data
data/temp/extract                        dedup     on             inherited from data
data/temp/public                         dedup     on             inherited from data
data/temp/rubbish                        dedup     on             inherited from data
data/temp/upload                         dedup     on             inherited from data
data/temp/xfer                           dedup     on             inherited from data
fast                                     dedup     on             local
fast/home                                dedup     on             inherited from fast
fast/home/jj5                            dedup     on             inherited from fast
fast/mysql                               dedup     off            local
fast/scratch                             dedup     off            local
fast/vbox                                dedup     on             inherited from fast
-------------------
Sun Dec 11 15:26:50 [bash:5.1.16 jobs:0 error:0 time:36]
root@longing:/home/jj5
# zfs get compression
NAME                                     PROPERTY     VALUE           SOURCE
data                                     compression  lz4             local
data/archive                             compression  lz4             inherited from data
data/backup                              compression  lz4             inherited from data
data/blob                                compression  lz4             inherited from data
data/dataset                             compression  lz4             inherited from data
data/host                                compression  lz4             inherited from data
data/image                               compression  off             local
data/image/archive                       compression  off             inherited from data/image
data/image/backup                        compression  off             inherited from data/image
data/image/vm                            compression  off             inherited from data/image
data/share                               compression  lz4             inherited from data
data/temp                                compression  lz4             inherited from data
data/temp/extract                        compression  lz4             inherited from data
data/temp/public                         compression  lz4             inherited from data
data/temp/rubbish                        compression  lz4             inherited from data
data/temp/upload                         compression  lz4             inherited from data
data/temp/xfer                           compression  lz4             inherited from data
fast                                     compression  lz4             local
fast/home                                compression  lz4             inherited from fast
fast/home/jj5                            compression  lz4             inherited from fast
fast/mysql                               compression  off             local
fast/scratch                             compression  off             local
fast/vbox                                compression  lz4             inherited from fast
-------------------

2023-01-06 jj5 - adding fast/session

# zpool upgrade fast
# zfs create fast/session
# zfs set compression=zstd fast/session
# zfs set dedup=off fast/session
# zfs set mountpoint=/var/log/session fast/session
# chmod 1777 /var/log/session