[ May 6 18:53:06 Enabled. ] [ May 6 18:53:06 Rereading configuration. ] [ May 6 18:53:10 Rereading configuration. ] [ May 6 18:53:17 Executing start method ("/opt/oxide/crucible/bin/crucible-agent run -D /opt/oxide/crucible/bin/crucible-downstairs --dataset oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible -l [fd00:1122:3344:101::13]:32345 -P 19000 -p downstairs -s snapshot"). ] note: configured to log to "/dev/stdout" {"msg":"dataset: \"oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.388739528Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834} {"msg":"listen IP: [fd00:1122:3344:101::13]:32345","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.390620554Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834} {"msg":"SMF instance name downstairs_prefix: \"downstairs\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.390637174Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834} {"msg":"Using conf_path:\"/data/crucible.json\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.406366801Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"listening","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:53:17.50462853Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.911636809Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:57837"} {"msg":"region 005d7663-55cb-45c4-97d0-6f2e38a642de state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.930887402Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.935843001Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"84cc37d4-4bcc-4413-969e-edb23e62a67f","remote_addr":"[fd00:1122:3344:101::a]:57837","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":13020,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.935863531Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.945758228Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:28.945786329Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"005d7663-55cb-45c4-97d0-6f2e38a642de\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19000, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.004754595Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"005d7663-55cb-45c4-97d0-6f2e38a642de","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"005d7663-55cb-45c4-97d0-6f2e38a642de\" \"--data\" \"/data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.006710432Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"005d7663-55cb-45c4-97d0-6f2e38a642de","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.12997879Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"005d7663-55cb-45c4-97d0-6f2e38a642de","component":"worker"} {"msg":"region 005d7663-55cb-45c4-97d0-6f2e38a642de state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.130034162Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.130129184Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.130608276Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.137181173Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.153563467Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.157012509Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.157773248Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.159754135Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.159810166Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.159847667Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.167986193Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.170320839Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"537a86a9-faa3-4724-a7e8-ab13e0160b77","remote_addr":"[fd00:1122:3344:101::a]:57837","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":859,"response_code":"200"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:54:29.172136322Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.495209599Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:62791"} {"msg":"region dec2f34f-7bef-4738-a326-dac235b9f135 state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.495334672Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.499043551Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"c4e86bfb-f107-4805-a083-6c753d968130","remote_addr":"[fd00:1122:3344:101::a]:62791","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":3761,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.499081662Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.509098572Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.509118632Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"dec2f34f-7bef-4738-a326-dac235b9f135\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19001, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/dec2f34f-7bef-4738-a326-dac235b9f135\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.567215267Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"dec2f34f-7bef-4738-a326-dac235b9f135","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"dec2f34f-7bef-4738-a326-dac235b9f135\" \"--data\" \"/data/regions/dec2f34f-7bef-4738-a326-dac235b9f135\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.568926668Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"dec2f34f-7bef-4738-a326-dac235b9f135","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.632710569Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"dec2f34f-7bef-4738-a326-dac235b9f135","component":"worker"} {"msg":"region dec2f34f-7bef-4738-a326-dac235b9f135 state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.63275124Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.632839742Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643315164Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643419226Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643517878Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643530329Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.64357579Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643635551Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643738744Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643818746Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643875517Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.643943159Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.650322302Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.650732242Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.653840176Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.664550533Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668433816Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/dec2f34f-7bef-4738-a326-dac235b9f135","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668730694Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19001","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668798285Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668842666Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.668882827Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.672140985Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.676183382Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:54.696168822Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"ed288d4e-ea92-41e6-abed-d0867ee60abb","remote_addr":"[fd00:1122:3344:101::a]:62791","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.369241848Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:60993"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.36930132Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","method":"GET","req_id":"6189f6d8-67d0-416a-8dcd-15a9eb7f2611","remote_addr":"[fd00:1122:3344:101::a]:60993","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":52,"response_code":"200"} {"msg":"path is \"/data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.369989716Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"dataset is oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.378516241Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"snapshot is oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.387058585Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.387085426Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"ZfsSnapshotInterface"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.402715451Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions/005d7663-55cb-45c4-97d0-6f2e38a642de/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","method":"GET","req_id":"a0ec139c-3b4f-45a9-b3ef-c4022e295d06","remote_addr":"[fd00:1122:3344:101::a]:60993","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":33246,"response_code":"200"} {"msg":"path is \"/data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.402887355Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"dataset is oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.411524233Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"snapshot is oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de@2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.419721809Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"ZfsSnapshotInterface"} {"msg":"parts is [\"oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/005d7663-55cb-45c4-97d0-6f2e38a642de\", \"2741d331-a7ac-4ab3-9b92-e5769c6b8cab\"]","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.419752Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"ZfsSnapshotInterface"} {"msg":"requesting running snapshot 005d7663-55cb-45c4-97d0-6f2e38a642de-2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.429099825Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"applying SMF actions for region 005d7663-55cb-45c4-97d0-6f2e38a642de running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab (state Requested)...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.429283449Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.42933554Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions/005d7663-55cb-45c4-97d0-6f2e38a642de/snapshots/2741d331-a7ac-4ab3-9b92-e5769c6b8cab/run","method":"POST","req_id":"77f75790-6b90-4b19-a057-019ccefdc66a","remote_addr":"[fd00:1122:3344:101::a]:60993","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":26472,"response_code":"200"} {"msg":"existing directory value /dev/null does not match /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.429856713Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.429976946Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430068518Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430087818Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430107129Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.43015552Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430512748Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.43057654Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430628921Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.430674532Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.434800341Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating missing snapshot instance snapshot-005d7663-55cb-45c4-97d0-6f2e38a642de-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.436199895Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:snapshot-005d7663-55cb-45c4-97d0-6f2e38a642de-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.443733506Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:snapshot-005d7663-55cb-45c4-97d0-6f2e38a642de-2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.454589456Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de/.zfs/snapshot/2741d331-a7ac-4ab3-9b92-e5769c6b8cab","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.454905614Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19002","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.454961465Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure mode SCF_TYPE_ASTRING ro","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.455002346Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.455040297Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.455086158Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.458586112Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.46264893Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"region 005d7663-55cb-45c4-97d0-6f2e38a642de running snapshot 2741d331-a7ac-4ab3-9b92-e5769c6b8cab state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:55.46265906Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"accepted connection","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.033853256Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","remote_addr":"[fd00:1122:3344:101::a]:40393"} {"msg":"region 42d90e75-13ae-485e-93cf-b977d8ee1abc state: Requested","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034572114Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034728747Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"5d16ed9e-131c-4763-8d30-95b67bc81604","remote_addr":"[fd00:1122:3344:101::a]:40393","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":852,"response_code":"200"} {"msg":"Region size:2147483648 reservation:2684354560 quota:6442450944","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.034747778Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set reservation of 2684354560 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.04442108Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"zfs set quota of 6442450944 for oxp_15a38827-a5e7-4eb0-96e1-7927042165c9/crucible/regions/42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.044445031Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating region Region { id: RegionId(\"42d90e75-13ae-485e-93cf-b977d8ee1abc\"), state: Requested, block_size: 512, extent_size: 131072, extent_count: 32, encrypted: true, port_number: 19003, cert_pem: None, key_pem: None, root_pem: None, source: None, read_only: false } at \"/data/regions/42d90e75-13ae-485e-93cf-b977d8ee1abc\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.102382121Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"42d90e75-13ae-485e-93cf-b977d8ee1abc","component":"worker"} {"msg":"downstairs create with: env -i \"/opt/oxide/crucible/bin/crucible-downstairs\" \"create\" \"--uuid\" \"42d90e75-13ae-485e-93cf-b977d8ee1abc\" \"--data\" \"/data/regions/42d90e75-13ae-485e-93cf-b977d8ee1abc\" \"--block-size\" \"512\" \"--extent-size\" \"131072\" \"--extent-count\" \"32\" \"--encrypted\"","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.102434713Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"42d90e75-13ae-485e-93cf-b977d8ee1abc","component":"worker"} {"msg":"region files created ok","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.166259794Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"region":"42d90e75-13ae-485e-93cf-b977d8ee1abc","component":"worker"} {"msg":"region 42d90e75-13ae-485e-93cf-b977d8ee1abc state: Requested -> Created","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.166312806Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"datafile"} {"msg":"applying SMF actions post create...","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.166390088Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing directory value /dev/null does not match /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167006843Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing port value 0 does not match 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167110515Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"existing address value 0.0.0.0 does not match fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167199127Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfig required","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167211808Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"using existing config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167248508Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.16730468Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/005d7663-55cb-45c4-97d0-6f2e38a642de","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167693649Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19000","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167770101Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167819372Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.167864613Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171418388Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating missing downstairs instance downstairs-42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.171849609Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok, have svc:/oxide/crucible/downstairs:downstairs-42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.174226126Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"creating config property group","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.180953407Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"reconfiguring svc:/oxide/crucible/downstairs:downstairs-42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186336656Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure directory SCF_TYPE_ASTRING /data/regions/42d90e75-13ae-485e-93cf-b977d8ee1abc","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186567742Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure port SCF_TYPE_COUNT 19003","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186635784Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ensure address SCF_TYPE_ASTRING fd00:1122:3344:101::13","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186678115Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"commit","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.186720636Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.190603889Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"SMF ok!","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.202247438Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"component":"worker"} {"msg":"request completed","v":0,"name":"crucible-agent","level":30,"time":"2024-05-06T18:59:58.275990548Z","hostname":"oxz_crucible_ec7f76c4-4eeb-40b9-ae42-1065d16d88ab","pid":22834,"uri":"/crucible/0/regions","method":"POST","req_id":"70fe65f2-d67e-4b70-8d7e-0a35335f465b","remote_addr":"[fd00:1122:3344:101::a]:40393","local_addr":"[fd00:1122:3344:101::13]:32345","component":"dropshot","latency_us":781,"response_code":"200"}